From 1887f9f516e10e8c7d11fe33800b3f7212bc706d Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 26 Feb 2018 13:27:34 +0530 Subject: [PATCH 01/38] initial cherry-pick --- .../java/io/druid/guice/GuiceInjectors.java | 1 + .../io/druid/guice/NullHandlingModule.java | 39 ++++++ .../io/druid/query/DefaultQueryMetrics.java | 2 + .../query/aggregation/AggregatorFactory.java | 8 +- .../query/aggregation/AggregatorUtil.java | 3 +- .../DoubleMaxAggregatorFactory.java | 37 +++++- .../DoubleMinAggregatorFactory.java | 37 +++++- .../DoubleSumAggregatorFactory.java | 31 ++++- .../FloatMaxAggregatorFactory.java | 34 ++++- .../FloatMinAggregatorFactory.java | 34 ++++- .../FloatSumAggregatorFactory.java | 28 +++- .../aggregation/LongMaxAggregatorFactory.java | 34 +++-- .../aggregation/LongMinAggregatorFactory.java | 34 +++-- .../aggregation/LongSumAggregatorFactory.java | 31 +++-- .../NullableAggregateCombiner.java | 111 ++++++++++++++++ .../query/aggregation/NullableAggregator.java | 104 +++++++++++++++ .../NullableAggregatorFactory.java | 72 ++++++++++ .../aggregation/NullableBufferAggregator.java | 114 ++++++++++++++++ .../query/aggregation/PostAggregator.java | 2 + .../SimpleDoubleAggregatorFactory.java | 8 +- .../SimpleFloatAggregatorFactory.java | 8 +- ...alityAggregatorColumnSelectorStrategy.java | 20 ++- ...alityAggregatorColumnSelectorStrategy.java | 15 ++- ...alityAggregatorColumnSelectorStrategy.java | 19 ++- ...alityAggregatorColumnSelectorStrategy.java | 36 +++-- .../first/DoubleFirstAggregatorFactory.java | 119 ++++++++++------- .../first/FloatFirstAggregatorFactory.java | 122 +++++++++-------- .../first/LongFirstAggregatorFactory.java | 125 ++++++++++-------- .../last/DoubleLastAggregatorFactory.java | 123 +++++++++-------- .../last/FloatLastAggregatorFactory.java | 123 +++++++++-------- .../last/LongLastAggregatorFactory.java | 125 ++++++++++-------- .../post/ArithmeticPostAggregator.java | 19 ++- .../post/DoubleGreatestPostAggregator.java | 25 ++-- .../post/DoubleLeastPostAggregator.java | 25 ++-- .../post/LongGreatestPostAggregator.java | 26 ++-- .../post/LongLeastPostAggregator.java | 26 ++-- .../dimension/ListFilteredDimensionSpec.java | 6 +- .../dimension/RegexFilteredDimensionSpec.java | 7 +- .../druid/query/expression/LikeExprMacro.java | 4 +- .../expression/RegexpExtractExprMacro.java | 7 +- .../extraction/FunctionalExtraction.java | 14 +- .../extraction/IdentityExtractionFn.java | 6 +- .../extraction/JavaScriptExtractionFn.java | 6 +- .../query/extraction/LowerExtractionFn.java | 5 +- .../query/extraction/MapLookupExtractor.java | 2 + .../extraction/MatchingDimExtractionFn.java | 4 +- .../extraction/RegexDimExtractionFn.java | 9 +- .../SearchQuerySpecDimExtractionFn.java | 4 +- .../extraction/StringFormatExtractionFn.java | 2 + .../query/extraction/StrlenExtractionFn.java | 4 + .../extraction/SubstringDimExtractionFn.java | 4 +- .../query/extraction/TimeDimExtractionFn.java | 4 +- .../query/extraction/UpperExtractionFn.java | 5 +- ...bleValueMatcherColumnSelectorStrategy.java | 29 ++-- .../query/filter/DruidDoublePredicate.java | 20 +++ .../query/filter/DruidFloatPredicate.java | 20 +++ .../query/filter/DruidLongPredicate.java | 20 +++ ...oatValueMatcherColumnSelectorStrategy.java | 29 ++-- .../io/druid/query/filter/InDimFilter.java | 48 ++++--- .../io/druid/query/filter/LikeDimFilter.java | 27 +--- ...ongValueMatcherColumnSelectorStrategy.java | 29 ++-- .../druid/query/filter/SelectorDimFilter.java | 29 +++- ...ingValueMatcherColumnSelectorStrategy.java | 36 ++--- .../io/druid/query/filter/ValueGetter.java | 3 + .../io/druid/query/groupby/GroupByQuery.java | 19 +-- .../epinephelinae/GroupByQueryEngineV2.java | 11 +- .../epinephelinae/RowBasedGrouperHelper.java | 117 ++++++++++------ ...ngStringGroupByColumnSelectorStrategy.java | 3 +- .../StringGroupByColumnSelectorStrategy.java | 3 +- .../groupby/having/EqualToHavingSpec.java | 4 + .../groupby/having/GreaterThanHavingSpec.java | 4 + .../groupby/having/LessThanHavingSpec.java | 4 + .../groupby/strategy/GroupByStrategyV2.java | 17 +-- .../io/druid/query/lookup/LookupConfig.java | 2 + .../druid/query/lookup/LookupExtractor.java | 3 +- .../druid/query/metadata/SegmentAnalyzer.java | 6 +- .../java/io/druid/query/search/SearchHit.java | 3 +- .../druid/query/search/SearchQueryRunner.java | 3 +- .../query/search/UseIndexesStrategy.java | 3 +- .../java/io/druid/query/topn/TopNMapFn.java | 12 +- .../ColumnSelectorBitmapIndexSelector.java | 2 +- .../druid/segment/DimensionHandlerUtils.java | 9 ++ .../druid/segment/DoubleDimensionIndexer.java | 4 +- .../druid/segment/FloatDimensionIndexer.java | 4 +- .../druid/segment/LongDimensionIndexer.java | 4 +- .../io/druid/segment/filter/BoundFilter.java | 10 +- .../segment/filter/ExpressionFilter.java | 4 +- .../java/io/druid/segment/filter/Filters.java | 4 +- .../io/druid/segment/filter/InFilter.java | 5 +- .../io/druid/segment/filter/LikeFilter.java | 27 ++-- .../serde/BitmapIndexColumnPartSupplier.java | 3 +- .../segment/serde/ComplexMetricSerde.java | 8 +- 92 files changed, 1679 insertions(+), 722 deletions(-) create mode 100644 processing/src/main/java/io/druid/guice/NullHandlingModule.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java create mode 100644 processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java diff --git a/processing/src/main/java/io/druid/guice/GuiceInjectors.java b/processing/src/main/java/io/druid/guice/GuiceInjectors.java index f3bac2362ce4..8adaae4dd25d 100644 --- a/processing/src/main/java/io/druid/guice/GuiceInjectors.java +++ b/processing/src/main/java/io/druid/guice/GuiceInjectors.java @@ -42,6 +42,7 @@ public static Collection makeDefaultStartupModules() new JacksonModule(), new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")), new ConfigModule(), + new NullHandlingModule(), new Module() { @Override diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java new file mode 100644 index 000000000000..7216ca363822 --- /dev/null +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.common.config.NullHandling; +import io.druid.common.config.NullValueHandlingConfig; + +/** + */ +public class NullHandlingModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.generic", NullValueHandlingConfig.class); + binder.requestStaticInjection(NullHandling.class); + binder.requestStaticInjection(NullHandling.class); + + } +} diff --git a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java index 8e13ae5bead1..b86ee99bb7d5 100644 --- a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java @@ -118,7 +118,9 @@ public void duration(QueryType query) @Override public void queryId(QueryType query) { + //CHECKSTYLE.OFF: Regexp setDimension(DruidMetrics.ID, Strings.nullToEmpty(query.getId())); + //CHECKSTYLE.ON: Regexp } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index d5e6790c9ab9..cadde722e50b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -36,6 +36,8 @@ * AggregatorFactory is a strategy (in the terms of Design Patterns) that represents column aggregation, e. g. min, * max, sum of metric columns, or cardinality of dimension columns (see {@link * io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory}). + * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * to extend {@link io.druid.query.aggregation.NullableAggregatorFactory}. */ @ExtensionPoint public abstract class AggregatorFactory implements Cacheable @@ -59,7 +61,8 @@ public abstract class AggregatorFactory implements Cacheable * * @return an object representing the combination of lhs and rhs, this can be a new object or a mutation of the inputs */ - public abstract Object combine(Object lhs, Object rhs); + @Nullable + public abstract Object combine(@Nullable Object lhs, @Nullable Object rhs); /** * Creates an AggregateCombiner to fold rollup aggregation results from serveral "rows" of different indexes during @@ -127,7 +130,8 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * * @return the finalized value that should be returned for the initial query */ - public abstract Object finalizeComputation(Object object); + @Nullable + public abstract Object finalizeComputation(@Nullable Object object); public abstract String getName(); diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 8a7477a73588..f1b2438fe02b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -37,6 +37,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.virtual.ExpressionSelectors; +import javax.annotation.Nullable; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -141,7 +142,7 @@ public static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefau final ExprMacroTable macroTable, final String fieldName, final String fieldExpression, - final float nullValue + @Nullable final Float nullValue ) { if (fieldName != null && fieldExpression == null) { diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 676647a21767..f23957dd2106 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -52,25 +56,46 @@ public DoubleMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleMaxAggregator(getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.NEGATIVE_INFINITY + ); + return Pair.of( + new DoubleMaxAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleMaxBufferAggregator(getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.NEGATIVE_INFINITY + ); + return Pair.of( + new DoubleMaxBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleMaxAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMaxAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index eb480a6397d6..7869bb1f2ce3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -51,25 +55,46 @@ public DoubleMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleMinAggregator(getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.POSITIVE_INFINITY + ); + return Pair.of( + new DoubleMinAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleMinBufferAggregator(getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.POSITIVE_INFINITY + ); + return Pair.of( + new DoubleMinBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleMinAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMinAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 34e398258d38..313a2676fdba 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -52,25 +56,40 @@ public DoubleSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleSumAggregator(getDoubleColumnSelector(metricFactory, 0.0)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); + return Pair.of( + new DoubleSumAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(getDoubleColumnSelector(metricFactory, 0.0)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); + return Pair.of( + new DoubleSumBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleSumAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleSumAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index ddd164cf4093..589c1def93e7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -51,25 +55,43 @@ public FloatMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatMaxAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.NEGATIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.NEGATIVE_INFINITY + ); + return Pair.of(new FloatMaxAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatMaxBufferAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.NEGATIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.NEGATIVE_INFINITY + ); + return Pair.of( + new FloatMaxBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatMaxAggregator.combineValues(finalizeComputation(lhs), finalizeComputation(rhs)); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMaxAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index 17c078c6cfa3..f97d7c5ba29f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -51,25 +55,43 @@ public FloatMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatMinAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.POSITIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.POSITIVE_INFINITY + ); + return Pair.of(new FloatMinAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatMinBufferAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.POSITIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.POSITIVE_INFINITY + ); + return Pair.of( + new FloatMinBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatMinAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMinAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index 2767a61d311f..b6c7725cf970 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -51,25 +55,37 @@ public FloatSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatSumAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); + return Pair.of(new FloatSumAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatSumBufferAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); + return Pair.of( + new FloatSumBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatSumAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleSumAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index cf2f88cf1905..5b63da651935 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -39,7 +42,7 @@ /** */ -public class LongMaxAggregatorFactory extends AggregatorFactory +public class LongMaxAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -72,15 +75,20 @@ public LongMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongMaxAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of(new LongMaxAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongMaxBufferAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of( + new LongMaxBufferAggregator(longColumnSelector), + longColumnSelector + ); } private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -101,13 +109,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return LongMaxAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new LongAggregateCombiner() { @@ -162,7 +177,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } @@ -215,7 +231,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index c7cee3131410..a1d556f8d349 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -23,13 +23,16 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -39,7 +42,7 @@ /** */ -public class LongMinAggregatorFactory extends AggregatorFactory +public class LongMinAggregatorFactory extends NullableAggregatorFactory { private final String name; @@ -73,15 +76,20 @@ public LongMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongMinAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of(new LongMinAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongMinBufferAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of( + new LongMinBufferAggregator(longColumnSelector), + longColumnSelector + ); } private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -102,13 +110,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return LongMinAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new LongAggregateCombiner() { @@ -163,7 +178,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } @@ -216,7 +232,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index bfc605b4a53d..44d0673b2a77 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -23,12 +23,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -38,7 +41,7 @@ /** */ -public class LongSumAggregatorFactory extends AggregatorFactory +public class LongSumAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -71,15 +74,20 @@ public LongSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongSumAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of(new LongSumAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of( + new LongSumBufferAggregator(longColumnSelector), + longColumnSelector + ); } private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -102,11 +110,17 @@ public Comparator getComparator() @Override public Object combine(Object lhs, Object rhs) { + if (lhs == null) { + return rhs; + } + if (rhs == null) { + return lhs; + } return LongSumAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new LongSumAggregateCombiner(); } @@ -140,7 +154,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } @@ -193,7 +208,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java new file mode 100644 index 000000000000..e33b7b458a4f --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -0,0 +1,111 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; + +/** + * The result of a NullableAggregateCombiner will be null if all the values to be combined are null values or no values are combined at all. + * If any of the value is non-null, the result would be the value of the delegate combiner. + * Note that the delegate combiner is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate combiner. + */ +public class NullableAggregateCombiner implements AggregateCombiner +{ + private boolean isNullResult = true; + + private final AggregateCombiner delegate; + + public NullableAggregateCombiner(AggregateCombiner delegate) + { + this.delegate = delegate; + } + + @Override + public void reset(ColumnValueSelector selector) + { + if (selector.isNull()) { + isNullResult = true; + } else { + isNullResult = false; + delegate.reset(selector); + } + } + + @Override + public void fold(ColumnValueSelector selector) + { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (isNullResult) { + isNullResult = false; + } + delegate.fold(selector); + } + } + + @Override + public float getFloat() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return primitive float for Null Value"); + } + return delegate.getFloat(); + } + + @Override + public double getDouble() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return double for Null Value"); + } + return delegate.getDouble(); + } + + @Override + public long getLong() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return long for Null Value"); + } + return delegate.getLong(); + } + + @Override + public boolean isNull() + { + return isNullResult || delegate.isNull(); + } + + @Nullable + @Override + public Object getObject() + { + return isNullResult ? null : delegate.getObject(); + } + + @Override + public Class classOfObject() + { + return delegate.classOfObject(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java new file mode 100644 index 000000000000..1eac57edc34b --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -0,0 +1,104 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.BaseNullableColumnValueSelector; + +import javax.annotation.Nullable; + +/** + * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. + * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. + * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate aggregator. + */ +public class NullableAggregator implements Aggregator +{ + private final Aggregator delegate; + private final BaseNullableColumnValueSelector selector; + private boolean isNullResult = true; + + public NullableAggregator(Aggregator delegate, BaseNullableColumnValueSelector selector) + { + this.delegate = delegate; + this.selector = selector; + } + + @Override + public void aggregate() + { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (isNullResult) { + isNullResult = false; + } + delegate.aggregate(); + } + } + + @Override + @Nullable + public Object get() + { + if (isNullResult) { + return null; + } + return delegate.get(); + } + + @Override + public float getFloat() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return float for Null Value"); + } + return delegate.getFloat(); + } + + @Override + public long getLong() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return long for Null Value"); + } + return delegate.getLong(); + } + + @Override + public double getDouble() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return double for Null Value"); + } + return delegate.getDouble(); + } + + @Override + public boolean isNull() + { + return isNullResult || delegate.isNull(); + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java new file mode 100644 index 000000000000..8999266181d7 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -0,0 +1,72 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + + +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.Pair; +import io.druid.segment.BaseNullableColumnValueSelector; +import io.druid.segment.ColumnSelectorFactory; + +/** + * abstract class with functionality to wrap aggregator/bufferAggregator/combiner to make them Nullable. + * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * to extend this class. + */ +public abstract class NullableAggregatorFactory extends AggregatorFactory +{ + @Override + public final Aggregator factorize(ColumnSelectorFactory metricFactory) + { + Pair pair = factorize2( + metricFactory); + return NullHandling.replaceWithDefault() ? pair.lhs : new NullableAggregator(pair.lhs, pair.rhs); + } + + protected abstract Pair factorize2(ColumnSelectorFactory metricfactory); + + @Override + public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + Pair pair = factorizeBuffered2( + metricFactory); + return NullHandling.replaceWithDefault() ? pair.lhs : new NullableBufferAggregator(pair.lhs, pair.rhs); + } + + protected abstract Pair factorizeBuffered2(ColumnSelectorFactory metricfactory); + + + @Override + public final AggregateCombiner makeAggregateCombiner() + { + AggregateCombiner combiner = makeAggregateCombiner2(); + return NullHandling.replaceWithDefault() ? combiner : new NullableAggregateCombiner(combiner); + } + + protected abstract AggregateCombiner makeAggregateCombiner2(); + + @Override + public final int getMaxIntermediateSize() + { + return getMaxIntermediateSize2() + (NullHandling.replaceWithDefault() ? 0 : Byte.BYTES); + } + + protected abstract int getMaxIntermediateSize2(); +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java new file mode 100644 index 000000000000..cd27bc6039d0 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -0,0 +1,114 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.BaseNullableColumnValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * The result of a NullableBufferAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. + * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. + * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate aggregator. + */ +public class NullableBufferAggregator implements BufferAggregator +{ + private static final byte IS_NULL_BYTE = (byte) 1; + private static final byte IS_NOT_NULL_BYTE = (byte) 0; + private final BufferAggregator delegate; + private final BaseNullableColumnValueSelector selector; + + + public NullableBufferAggregator(BufferAggregator delegate, BaseNullableColumnValueSelector selector) + { + this.delegate = delegate; + this.selector = selector; + } + + @Override + public void init(ByteBuffer buf, int position) + { + buf.put(position, IS_NULL_BYTE); + delegate.init(buf, position + Byte.BYTES); + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (buf.get(position) == IS_NULL_BYTE) { + buf.put(position, IS_NOT_NULL_BYTE); + } + delegate.aggregate(buf, position + Byte.BYTES); + } + } + + @Override + @Nullable + public Object get(ByteBuffer buf, int position) + { + if (buf.get(position) == IS_NULL_BYTE) { + return null; + } + return delegate.get(buf, position + Byte.BYTES); + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return float for Null Value"); + } + return delegate.getFloat(buf, position + Byte.BYTES); + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return long for Null Value"); + } + return delegate.getLong(buf, position + Byte.BYTES); + } + + @Override + public double getDouble(ByteBuffer buf, int position) + { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return double for Null Value"); + } + return delegate.getDouble(buf, position + Byte.BYTES); + } + + @Override + public boolean isNull(ByteBuffer buf, int position) + { + return buf.get(position) == IS_NULL_BYTE || delegate.isNull(buf, position + Byte.BYTES); + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java index 171d4f39ef18..3d789d1d3dab 100644 --- a/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java @@ -22,6 +22,7 @@ import io.druid.guice.annotations.ExtensionPoint; import io.druid.java.util.common.Cacheable; +import javax.annotation.Nullable; import java.util.Comparator; import java.util.Map; import java.util.Set; @@ -36,6 +37,7 @@ public interface PostAggregator extends Cacheable Comparator getComparator(); + @Nullable Object compute(Map combinedAggregators); String getName(); diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index f3b173805f0a..6a809a441695 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -28,12 +28,13 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; -public abstract class SimpleDoubleAggregatorFactory extends AggregatorFactory +public abstract class SimpleDoubleAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -91,7 +92,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Double.BYTES; } @@ -152,7 +153,8 @@ public List requiredFields() } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index a37e0226e0dd..cdc797658577 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -26,12 +26,13 @@ import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; -public abstract class SimpleFloatAggregatorFactory extends AggregatorFactory +public abstract class SimpleFloatAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -87,7 +88,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Float.BYTES; } @@ -99,7 +100,8 @@ public Comparator getComparator() } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java index 5fd15ae30c1b..3874c1e4250c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java @@ -20,23 +20,33 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseDoubleColumnValueSelector; - +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling#replaceWithDefault()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class DoubleCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override - public void hashRow(BaseDoubleColumnValueSelector dimSelector, Hasher hasher) + public void hashRow(BaseDoubleColumnValueSelector selector, Hasher hasher) { - hasher.putDouble(dimSelector.getDouble()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + hasher.putDouble(selector.getDouble()); + } } @Override - public void hashValues(BaseDoubleColumnValueSelector dimSelector, HyperLogLogCollector collector) + public void hashValues(BaseDoubleColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(dimSelector.getDouble())).asBytes()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(selector.getDouble())).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java index b46261c7b15e..995667f94237 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java @@ -20,22 +20,33 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseFloatColumnValueSelector; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling#replaceWithDefault()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class FloatCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override public void hashRow(BaseFloatColumnValueSelector selector, Hasher hasher) { - hasher.putFloat(selector.getFloat()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + hasher.putFloat(selector.getFloat()); + } } @Override public void hashValues(BaseFloatColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java index a666ed64e1a7..75dba24c4568 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java @@ -20,22 +20,33 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseLongColumnValueSelector; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling#replaceWithDefault()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class LongCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override - public void hashRow(BaseLongColumnValueSelector dimSelector, Hasher hasher) + public void hashRow(BaseLongColumnValueSelector selector, Hasher hasher) { - hasher.putLong(dimSelector.getLong()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + hasher.putLong(selector.getLong()); + } } @Override - public void hashValues(BaseLongColumnValueSelector dimSelector, HyperLogLogCollector collector) + public void hashValues(BaseLongColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashLong(dimSelector.getLong()).asBytes()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(selector.getLong()).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java index 71661abf91b9..036d07d1e967 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.DimensionSelector; @@ -40,20 +41,34 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) // nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases. if (size == 1) { final String value = dimSelector.lookupName(row.get(0)); - hasher.putUnencodedChars(nullToSpecial(value)); + if (NullHandling.replaceWithDefault() || value != null) { + hasher.putUnencodedChars(nullToSpecial(value)); + } } else if (size != 0) { + boolean hasNonNullValue = false; final String[] values = new String[size]; for (int i = 0; i < size; ++i) { final String value = dimSelector.lookupName(row.get(i)); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. + if (!NullHandling.replaceWithDefault() && !hasNonNullValue && value != null) { + hasNonNullValue = true; + } values[i] = nullToSpecial(value); } - // Values need to be sorted to ensure consistent multi-value ordering across different segments - Arrays.sort(values); - for (int i = 0; i < size; ++i) { - if (i != 0) { - hasher.putChar(CARDINALITY_AGG_SEPARATOR); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. + if (NullHandling.replaceWithDefault() || hasNonNullValue) { + // Values need to be sorted to ensure consistent multi-value ordering across different segments + Arrays.sort(values); + for (int i = 0; i < size; ++i) { + if (i != 0) { + hasher.putChar(CARDINALITY_AGG_SEPARATOR); + } + hasher.putUnencodedChars(values[i]); } - hasher.putUnencodedChars(values[i]); } } } @@ -65,7 +80,12 @@ public void hashValues(DimensionSelector dimSelector, HyperLogLogCollector colle for (int i = 0; i < row.size(); i++) { int index = row.get(i); final String value = dimSelector.lookupName(index); - collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. + if (NullHandling.replaceWithDefault() || value != null) { + collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index bc725cc0769a..0fc65fcb5e4d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -25,6 +25,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -32,11 +33,15 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleFirstAggregatorFactory extends AggregatorFactory +public class DoubleFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, @@ -75,20 +80,26 @@ public DoubleFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) + BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) + BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector ); } @@ -99,13 +110,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("DoubleFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -116,46 +134,48 @@ public AggregatorFactory getCombiningFactory() return new DoubleFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Long.BYTES, pair.rhs); - } - } + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Long.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -174,9 +194,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -219,7 +240,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Double.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index fe66d280ca5a..775ec3a49eb0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -25,6 +25,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -32,11 +33,14 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +48,7 @@ import java.util.Map; import java.util.Objects; -public class FloatFirstAggregatorFactory extends AggregatorFactory +public class FloatFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, @@ -73,21 +77,25 @@ public FloatFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector); } @Override @@ -97,13 +105,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("FloatFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -114,46 +129,48 @@ public AggregatorFactory getCombiningFactory() return new FloatFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -172,9 +189,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? object : ((SerializablePair) object).rhs; } @Override @@ -214,7 +232,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Float.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index a2780e94e6a4..b6e7634d40fe 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -23,26 +23,31 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; -public class LongFirstAggregatorFactory extends AggregatorFactory +public class LongFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Longs.compare( ((SerializablePair) o1).rhs, @@ -66,21 +71,25 @@ public LongFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override @@ -90,13 +99,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (lhs == null) { + return rhs; + } + if (rhs == null) { + return lhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("LongFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -107,46 +123,48 @@ public AggregatorFactory getCombiningFactory() return new LongFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -165,9 +183,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? object : ((SerializablePair) object).rhs; } @Override @@ -207,7 +226,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES * 2; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index ed011684f06f..09756e2c5b0f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -30,13 +31,17 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleLastAggregatorFactory extends AggregatorFactory +public class DoubleLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; @@ -65,21 +70,25 @@ public DoubleLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + doubleColumnSelector + ), doubleColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + doubleColumnSelector + ), doubleColumnSelector); } @Override @@ -89,13 +98,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("DoubleLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -106,46 +122,48 @@ public AggregatorFactory getCombiningFactory() return new DoubleLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -164,9 +182,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -210,7 +229,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Double.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 9dce9c8949ab..5f28ac48dc0a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -30,13 +31,17 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.FloatFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class FloatLastAggregatorFactory extends AggregatorFactory +public class FloatLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; @@ -63,21 +68,25 @@ public FloatLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + floatColumnSelector + ), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + floatColumnSelector + ), floatColumnSelector); } @Override @@ -87,13 +96,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("FloatLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -104,46 +120,48 @@ public AggregatorFactory getCombiningFactory() return new FloatLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -162,9 +180,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -205,7 +224,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Float.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 4bee9e9a6659..6af3336d8c1a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -22,21 +22,26 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class LongLastAggregatorFactory extends AggregatorFactory +public class LongLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; private final String name; @@ -62,21 +67,25 @@ public LongLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override @@ -86,13 +95,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("LongLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -103,46 +119,48 @@ public AggregatorFactory getCombiningFactory() return new LongLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -161,9 +179,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -203,7 +222,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES * 2; } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index 5e2bd5b10149..ecb70da4bf16 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; @@ -109,11 +110,21 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = 0.0; + Double retVal = NullHandling.defaultDoubleValue(); if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); + Number nextVal = (Number) fieldsIter.next().compute(values); + if (nextVal == null) { + // As per SQL standard if any of the value is null, arithmetic operators will return null. + return null; + } + retVal = nextVal.doubleValue(); while (fieldsIter.hasNext()) { - retVal = op.compute(retVal, ((Number) fieldsIter.next().compute(values)).doubleValue()); + nextVal = (Number) fieldsIter.next().compute(values); + if (nextVal == null) { + // As per SQL standard if any of the value is null, arithmetic operators will return null. + return null; + } + retVal = op.compute(retVal, (nextVal).doubleValue()); } } return retVal; @@ -268,7 +279,7 @@ public enum Ordering implements Comparator /** * Ensures the following order: numeric > NaN > Infinite. * - * The name may be referenced via Ordering.valueOf(String) in the constructor {@link + * The name may be referenced via {@link #valueOf(String)} in the constructor {@link * ArithmeticPostAggregator#ArithmeticPostAggregator(String, String, List, String)}. */ @SuppressWarnings("unused") diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 584d022675b5..7e0100028e8e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -37,14 +39,7 @@ public class DoubleGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return ((Double) o).compareTo((Double) o1); - } - }; + private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); private final String name; private final List fields; @@ -81,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = Double.NEGATIVE_INFINITY; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); - while (fieldsIter.hasNext()) { - double other = ((Number) fieldsIter.next().compute(values)).doubleValue(); - if (other > retVal) { - retVal = other; - } + Double retVal = NullHandling.replaceWithDefault() ? Double.NEGATIVE_INFINITY : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) > 0) { + retVal = nextVal.doubleValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index f66c81d9e769..583328a4249c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -22,7 +22,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -37,14 +39,7 @@ public class DoubleLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return ((Double) o).compareTo((Double) o1); - } - }; + private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); private final String name; private final List fields; @@ -81,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = Double.POSITIVE_INFINITY; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); - while (fieldsIter.hasNext()) { - double other = ((Number) fieldsIter.next().compute(values)).doubleValue(); - if (other < retVal) { - retVal = other; - } + Double retVal = NullHandling.replaceWithDefault() ? Double.POSITIVE_INFINITY : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) < 0) { + retVal = nextVal.doubleValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index 695568c4dad4..69f439aacb93 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -23,7 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -38,14 +39,7 @@ public class LongGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((Number) o).longValue(), ((Number) o1).longValue()); - } - }; + private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); private final String name; private final List fields; @@ -82,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - long retVal = Long.MIN_VALUE; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).longValue(); - while (fieldsIter.hasNext()) { - long other = ((Number) fieldsIter.next().compute(values)).longValue(); - if (other > retVal) { - retVal = other; - } + Long retVal = NullHandling.replaceWithDefault() ? Long.MIN_VALUE : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) > 0) { + retVal = nextVal.longValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index f14bcd8228f3..01835ed2e34d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -22,8 +22,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -38,14 +39,7 @@ public class LongLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((Number) o).longValue(), ((Number) o1).longValue()); - } - }; + private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); private final String name; private final List fields; @@ -82,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - long retVal = Long.MAX_VALUE; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).longValue(); - while (fieldsIter.hasNext()) { - long other = ((Number) fieldsIter.next().compute(values)).longValue(); - if (other < retVal) { - retVal = other; - } + Long retVal = NullHandling.replaceWithDefault() ? Long.MAX_VALUE : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) < 0) { + retVal = nextVal.longValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index c7ab3873b578..3fb3e852d676 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; @@ -106,7 +106,7 @@ private DimensionSelector filterWhiteList(DimensionSelector selector) } } else { for (int i = 0; i < selectorCardinality; i++) { - if (values.contains(Strings.nullToEmpty(selector.lookupName(i)))) { + if (values.contains(NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } @@ -137,7 +137,7 @@ public boolean apply(@Nullable String input) forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; for (int i = 0; i < selectorCardinality; i++) { - if (!values.contains(Strings.nullToEmpty(selector.lookupName(i)))) { + if (!values.contains(NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index 4ebd407083f4..ce11703476be 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; @@ -76,7 +76,7 @@ public DimensionSelector decorate(final DimensionSelector selector) @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(Strings.nullToEmpty(input)).matches(); + return compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches(); } } ); @@ -86,7 +86,8 @@ public boolean apply(@Nullable String input) final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(); forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { - if (compiledRegex.matcher(Strings.nullToEmpty(selector.lookupName(i))).matches()) { + String val = NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)); + if (val != null && compiledRegex.matcher(val).matches()) { forwardMapping.put(i, count++); } } diff --git a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java index c60400be33eb..d03ea8d13c7d 100644 --- a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java @@ -19,7 +19,7 @@ package io.druid.query.expression; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -63,7 +63,7 @@ public Expr apply(final List args) } final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from( - Strings.nullToEmpty((String) patternExpr.getLiteralValue()), + NullHandling.nullToEmptyIfNeeded((String) patternExpr.getLiteralValue()), escapeChar ); diff --git a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java index f9a4273a05ae..940630c250c6 100644 --- a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java @@ -19,7 +19,7 @@ package io.druid.query.expression; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -63,9 +63,10 @@ class RegexpExtractExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - final Matcher matcher = pattern.matcher(Strings.nullToEmpty(arg.eval(bindings).asString())); + String s = arg.eval(bindings).asString(); + final Matcher matcher = pattern.matcher(NullHandling.nullToEmptyIfNeeded(s)); final String retVal = matcher.find() ? matcher.group(index) : null; - return ExprEval.of(Strings.emptyToNull(retVal)); + return ExprEval.of(NullHandling.emptyToNullIfNeeded(retVal)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java index bd762574efd1..1bacaab5ec52 100644 --- a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java +++ b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java @@ -21,7 +21,7 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -52,9 +52,9 @@ public FunctionalExtraction( ) { this.retainMissingValue = retainMissingValue; - this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandling.emptyToNullIfNeeded(replaceMissingValueWith); Preconditions.checkArgument( - !(this.retainMissingValue && !Strings.isNullOrEmpty(this.replaceMissingValueWith)), + !(this.retainMissingValue && !(this.replaceMissingValueWith == null)), "Cannot specify a [replaceMissingValueWith] and set [retainMissingValue] to true" ); @@ -69,7 +69,7 @@ public FunctionalExtraction( public String apply(@Nullable String dimValue) { final String retval = extractionFunction.apply(dimValue); - return Strings.isNullOrEmpty(retval) ? Strings.emptyToNull(dimValue) : retval; + return NullHandling.isNullOrEquivalent(retval) ? NullHandling.emptyToNullIfNeeded(dimValue) : retval; } }; } else { @@ -79,8 +79,10 @@ public String apply(@Nullable String dimValue) @Override public String apply(@Nullable String dimValue) { - final String retval = extractionFunction.apply(dimValue); - return Strings.isNullOrEmpty(retval) ? FunctionalExtraction.this.replaceMissingValueWith : retval; + final String retval = NullHandling.emptyToNullIfNeeded(extractionFunction.apply(dimValue)); + return retval == null + ? FunctionalExtraction.this.replaceMissingValueWith + : retval; } }; } diff --git a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java index 1bf700feaaf0..0967bdfac0ad 100644 --- a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java @@ -19,7 +19,7 @@ package io.druid.query.extraction; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -42,14 +42,14 @@ public byte[] getCacheKey() @Nullable public String apply(@Nullable Object value) { - return value == null ? null : Strings.emptyToNull(value.toString()); + return value == null ? null : NullHandling.emptyToNullIfNeeded(value.toString()); } @Override @Nullable public String apply(@Nullable String value) { - return Strings.emptyToNull(value); + return NullHandling.emptyToNullIfNeeded(value); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java index 1d4e479acd8d..fa7929f8ce79 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; import org.mozilla.javascript.Context; @@ -113,7 +113,7 @@ public byte[] getCacheKey() public String apply(@Nullable Object value) { checkAndCompileScript(); - return Strings.emptyToNull(fn.apply(value)); + return NullHandling.emptyToNullIfNeeded(fn.apply(value)); } /** @@ -139,7 +139,7 @@ private void checkAndCompileScript() @Nullable public String apply(@Nullable String value) { - return this.apply((Object) Strings.emptyToNull(value)); + return this.apply((Object) NullHandling.emptyToNullIfNeeded(value)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index da2a8e3bc702..b0dddf121b73 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -52,7 +53,7 @@ public LowerExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (Strings.isNullOrEmpty(key)) { + if (NullHandling.isNullOrEquivalent(key)) { return null; } return key.toLowerCase(locale); @@ -73,7 +74,9 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { + //CHECKSTYLE.OFF: Regexp byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_LOWER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index b818bd8cd50c..ccdae8f0932f 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -76,7 +76,9 @@ public List unapply(final String value) { @Override public boolean apply(@Nullable String key) { + //CHECKSTYLE.OFF: Regexp return map.get(key).equals(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp } }).keySet()); diff --git a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java index 4d121be031fc..5c1054afad91 100644 --- a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -62,7 +62,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { // We'd return null whether or not the pattern matched return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index 3a1b0a5cc293..feab409d4ba1 100644 --- a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -106,13 +106,14 @@ public byte[] getCacheKey() public String apply(@Nullable String dimValue) { final String retVal; - final Matcher matcher = pattern.matcher(Strings.nullToEmpty(dimValue)); - if (matcher.find()) { + String val = NullHandling.nullToEmptyIfNeeded(dimValue); + final Matcher matcher = val == null ? null : pattern.matcher(val); + if (matcher != null && matcher.find()) { retVal = matcher.group(index); } else { retVal = replaceMissingValue ? replaceMissingValueWith : dimValue; } - return Strings.emptyToNull(retVal); + return NullHandling.emptyToNullIfNeeded(retVal); } @JsonProperty("expr") diff --git a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java index 1f005523925b..3e30daa6f5f6 100644 --- a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.query.search.SearchQuerySpec; import javax.annotation.Nullable; @@ -64,7 +64,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - return searchQuerySpec.accept(dimValue) ? Strings.emptyToNull(dimValue) : null; + return searchQuerySpec.accept(dimValue) ? NullHandling.emptyToNullIfNeeded(dimValue) : null; } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java index 168cad01ea24..ba21754774fe 100644 --- a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java @@ -107,7 +107,9 @@ public String apply(@Nullable String value) value = ""; } } + //CHECKSTYLE.OFF: Regexp return Strings.emptyToNull(StringUtils.format(format, value)); + //CHECKSTYLE.ON: Regexp } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java index 5ad88fa38fce..780f1ba7afe4 100644 --- a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java @@ -20,6 +20,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -40,6 +41,9 @@ public static StrlenExtractionFn instance() @Override public String apply(@Nullable String value) { + if (!NullHandling.replaceWithDefault() && value == null) { + return null; + } return String.valueOf(value == null ? 0 : value.length()); } diff --git a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java index 75b251f18faf..c980aaa920fb 100644 --- a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -63,7 +63,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 0aa7fdeb8acf..c69a21558a68 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.ibm.icu.text.SimpleDateFormat; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -77,7 +77,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index 34368f7f0fd6..ec2b914d3806 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -51,7 +52,7 @@ public UpperExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (Strings.isNullOrEmpty(key)) { + if (NullHandling.isNullOrEquivalent(key)) { return null; } return key.toUpperCase(locale); @@ -72,7 +73,9 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { + //CHECKSTYLE.OFF: Regexp byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_UPPER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java index f41e6c6a0242..a99d8a52de4f 100644 --- a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.filter.BooleanValueMatcher; public class DoubleValueMatcherColumnSelectorStrategy @@ -33,7 +32,20 @@ public ValueMatcher makeValueMatcher(final BaseDoubleColumnValueSelector selecto { final Double matchVal = DimensionHandlerUtils.convertObjectToDouble(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final long matchValLongBits = Double.doubleToLongBits(matchVal); @@ -65,6 +77,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyDouble(selector.getDouble()); } @@ -80,13 +95,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final BaseDoubleColumnValueSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Double.toString(selector.getDouble())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Double.toString(selector.getDouble())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index c28b8d376873..f1e2874ea6ee 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -26,5 +26,25 @@ public interface DruidDoublePredicate DruidDoublePredicate ALWAYS_TRUE = input -> true; + DruidDoublePredicate MATCH_NULL_ONLY = new DruidDoublePredicate() + { + @Override + public boolean applyDouble(double input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyDouble(double input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index 2aa2674dd246..63de55c2f9f0 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -26,5 +26,25 @@ public interface DruidFloatPredicate { DruidFloatPredicate ALWAYS_FALSE = input -> false; + DruidFloatPredicate MATCH_NULL_ONLY = new DruidFloatPredicate() + { + @Override + public boolean applyFloat(float input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyFloat(float input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java index 2afca5c40e10..4175c6bcbe18 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java @@ -28,5 +28,25 @@ public interface DruidLongPredicate DruidLongPredicate ALWAYS_TRUE = input -> true; + DruidLongPredicate MATCH_NULL_ONLY = new DruidLongPredicate() + { + @Override + public boolean applyLong(long input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyLong(long input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java index af7a2b215ed9..5628e09bf92d 100644 --- a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.filter.BooleanValueMatcher; public class FloatValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy @@ -32,7 +31,20 @@ public ValueMatcher makeValueMatcher(final BaseFloatColumnValueSelector selector { final Float matchVal = DimensionHandlerUtils.convertObjectToFloat(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final int matchValIntBits = Float.floatToIntBits(matchVal); @@ -64,6 +76,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyFloat(selector.getFloat()); } @@ -79,13 +94,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final BaseFloatColumnValueSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Float.toString(selector.getFloat())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Float.toString(selector.getFloat())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index d12d0a552742..4e56d7eda92c 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -21,19 +21,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.extraction.ExtractionFn; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; @@ -51,6 +51,8 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; public class InDimFilter implements DimFilter { @@ -58,7 +60,7 @@ public class InDimFilter implements DimFilter // Hashing threshold is not applied to String for now, String still uses ImmutableSortedSet public static final int NUMERIC_HASHING_THRESHOLD = 16; - private final ImmutableSortedSet values; + private final SortedSet values; private final String dimension; private final ExtractionFn extractionFn; private final Supplier longPredicateSupplier; @@ -74,19 +76,11 @@ public InDimFilter( { Preconditions.checkNotNull(dimension, "dimension can not be null"); Preconditions.checkArgument(values != null && !values.isEmpty(), "values can not be null or empty"); - this.values = ImmutableSortedSet.copyOf( - Iterables.transform( - values, new Function() - { - @Override - public String apply(String input) - { - return Strings.nullToEmpty(input); - } - } - ) - ); + this.values = new TreeSet<>(Comparators.naturalNullsFirst()); + for (String value : values) { + this.values.add(NullHandling.emptyToNullIfNeeded(value)); + } this.dimension = dimension; this.extractionFn = extractionFn; this.longPredicateSupplier = getLongPredicateSupplier(); @@ -119,14 +113,20 @@ public byte[] getCacheKey() final byte[][] valuesBytes = new byte[values.size()][]; int valuesBytesSize = 0; int index = 0; + boolean hasNull = false; for (String value : values) { + if (value == null) { + hasNull = true; + } + //CHECKSTYLE.OFF: Regexp valuesBytes[index] = StringUtils.toUtf8(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp valuesBytesSize += valuesBytes[index].length + 1; ++index; } byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - ByteBuffer filterCacheKey = ByteBuffer.allocate(3 + ByteBuffer filterCacheKey = ByteBuffer.allocate(5 + dimensionBytes.length + valuesBytesSize + extractionFnBytes.length) @@ -134,6 +134,8 @@ public byte[] getCacheKey() .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(hasNull ? (byte) 1 : (byte) 0) .put(DimFilterUtils.STRING_SEPARATOR); for (byte[] bytes : valuesBytes) { filterCacheKey.put(bytes) @@ -165,7 +167,9 @@ private InDimFilter optimizeLookup() // We cannot do an unapply()-based optimization if the selector value // and the replaceMissingValuesWith value are the same, since we have to match on // all values that are not present in the lookup. + //CHECKSTYLE.OFF: Regexp final String convertedValue = Strings.emptyToNull(value); + //CHECKSTYLE.ON: Regexp if (!exFn.isRetainMissingValue() && Objects.equals(convertedValue, exFn.getReplaceMissingValueWith())) { return this; } @@ -175,7 +179,7 @@ private InDimFilter optimizeLookup() // there may be row values that match the selector value but are not included // in the lookup map. Match on the selector value as well. // If the selector value is overwritten in the lookup map, don't add selector value to keys. - if (exFn.isRetainMissingValue() && lookup.apply(convertedValue) == null) { + if (exFn.isRetainMissingValue() && NullHandling.isNullOrEquivalent(lookup.apply(convertedValue))) { keys.add(convertedValue); } } @@ -210,7 +214,9 @@ public RangeSet getDimensionRangeSet(String dimension) } RangeSet retSet = TreeRangeSet.create(); for (String value : values) { + //CHECKSTYLE.OFF: Regexp retSet.add(Range.singleton(Strings.nullToEmpty(value))); + //CHECKSTYLE.ON: Regexp } return retSet; } @@ -260,9 +266,11 @@ public String toString() if (extractionFn != null) { builder.append(")"); } - - builder.append(" IN (").append(Joiner.on(", ").join(values)).append(")"); - + //CHECKSTYLE.OFF: Regexp + builder.append(" IN (") + .append(Joiner.on(", ").join(Iterables.transform(values, input -> Strings.nullToEmpty(input)))) + .append(")"); + //CHECKSTYLE.ON: Regexp return builder.toString(); } diff --git a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java index 5a98bc5d2327..8a55ffbd4a68 100644 --- a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java @@ -23,10 +23,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.collect.RangeSet; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.data.Indexed; @@ -98,7 +98,7 @@ private LikeMatcher( ) { this.suffixMatch = Preconditions.checkNotNull(suffixMatch, "suffixMatch"); - this.prefix = Strings.nullToEmpty(prefix); + this.prefix = NullHandling.nullToEmptyIfNeeded(prefix); this.pattern = Preconditions.checkNotNull(pattern, "pattern"); } @@ -151,7 +151,8 @@ private static void addPatternCharacter(final StringBuilder patternBuilder, fina public boolean matches(@Nullable final String s) { - return pattern.matcher(Strings.nullToEmpty(s)).matches(); + String val = NullHandling.nullToEmptyIfNeeded(s); + return val != null && pattern.matcher(val).matches(); } /** @@ -165,7 +166,7 @@ public boolean matchesSuffixOnly(final Indexed strings, final int i) return true; } else if (suffixMatch == SuffixMatch.MATCH_EMPTY) { final String s = strings.get(i); - return (s == null ? 0 : s.length()) == prefix.length(); + return s == null ? matches(null) : s.length() == prefix.length(); } else { // suffixMatch is MATCH_PATTERN final String s = strings.get(i); @@ -181,23 +182,9 @@ public DruidPredicateFactory predicateFactory(final ExtractionFn extractionFn) public Predicate makeStringPredicate() { if (extractionFn != null) { - return new Predicate() - { - @Override - public boolean apply(String input) - { - return matches(extractionFn.apply(input)); - } - }; + return input -> matches(extractionFn.apply(input)); } else { - return new Predicate() - { - @Override - public boolean apply(String input) - { - return matches(input); - } - }; + return input -> matches(input); } } diff --git a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java index 15a62bd50631..9b7966d7dd8f 100644 --- a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.filter.BooleanValueMatcher; public class LongValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy @@ -32,7 +31,20 @@ public ValueMatcher makeValueMatcher(final BaseLongColumnValueSelector selector, { final Long matchVal = DimensionHandlerUtils.convertObjectToLong(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final long matchValLong = matchVal; return new ValueMatcher() @@ -63,6 +75,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyLong(selector.getLong()); } @@ -78,13 +93,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final BaseLongColumnValueSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Long.toString(selector.getLong())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Long.toString(selector.getLong())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index d13971a8ea28..8c18a99ccbba 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -25,12 +25,12 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; @@ -38,6 +38,7 @@ import io.druid.segment.filter.SelectorFilter; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Objects; /** @@ -64,7 +65,7 @@ public SelectorDimFilter( Preconditions.checkArgument(dimension != null, "dimension must not be null"); this.dimension = dimension; - this.value = Strings.nullToEmpty(value); + this.value = NullHandling.emptyToNullIfNeeded(value); this.extractionFn = extractionFn; } @@ -75,8 +76,10 @@ public byte[] getCacheKey() byte[] valueBytes = (value == null) ? new byte[]{} : StringUtils.toUtf8(value); byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - return ByteBuffer.allocate(3 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) + return ByteBuffer.allocate(6 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) .put(DimFilterUtils.SELECTOR_CACHE_ID) + .put(value == null ? (byte) 1 : (byte) 0) + .put(DimFilterUtils.STRING_SEPARATOR) .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(valueBytes) @@ -88,7 +91,7 @@ public byte[] getCacheKey() @Override public DimFilter optimize() { - return new InDimFilter(dimension, ImmutableList.of(value), extractionFn).optimize(); + return new InDimFilter(dimension, Arrays.asList(value), extractionFn).optimize(); } @Override @@ -97,14 +100,13 @@ public Filter toFilter() if (extractionFn == null) { return new SelectorFilter(dimension, value); } else { - final String valueOrNull = Strings.emptyToNull(value); final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override public Predicate makeStringPredicate() { - return Predicates.equalTo(valueOrNull); + return Predicates.equalTo(value); } @Override @@ -188,7 +190,9 @@ public RangeSet getDimensionRangeSet(String dimension) return null; } RangeSet retSet = TreeRangeSet.create(); + //CHECKSTYLE.OFF: Regexp retSet.add(Range.singleton(Strings.nullToEmpty(value))); + //CHECKSTYLE.ON: Regexp return retSet; } @@ -211,6 +215,10 @@ private void initLongPredicate() if (longPredicate != null) { return; } + if (value == null) { + longPredicate = DruidLongPredicate.MATCH_NULL_ONLY; + return; + } final Long valueAsLong = GuavaUtils.tryParseLong(value); if (valueAsLong == null) { longPredicate = DruidLongPredicate.ALWAYS_FALSE; @@ -231,6 +239,11 @@ private void initFloatPredicate() if (floatPredicate != null) { return; } + + if (value == null) { + floatPredicate = DruidFloatPredicate.MATCH_NULL_ONLY; + return; + } final Float valueAsFloat = Floats.tryParse(value); if (valueAsFloat == null) { @@ -251,6 +264,10 @@ private void initDoublePredicate() if (druidDoublePredicate != null) { return; } + if (value == null) { + druidDoublePredicate = DruidDoublePredicate.MATCH_NULL_ONLY; + return; + } final Double aDouble = Doubles.tryParse(value); if (aDouble == null) { diff --git a/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java index e380b1162ee3..aa3dfe14aa4f 100644 --- a/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java @@ -20,7 +20,6 @@ package io.druid.query.filter; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; @@ -28,19 +27,11 @@ public class StringValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy { private static final String[] NULL_VALUE = new String[]{null}; - private static final ValueGetter NULL_VALUE_GETTER = new ValueGetter() - { - @Override - public String[] get() - { - return NULL_VALUE; - } - }; + private static final ValueGetter NULL_VALUE_GETTER = () -> NULL_VALUE; @Override public ValueMatcher makeValueMatcher(final DimensionSelector selector, String value) { - value = Strings.emptyToNull(value); if (selector.getValueCardinality() == 0) { return BooleanValueMatcher.of(value == null); } else { @@ -68,22 +59,17 @@ public ValueGetter makeValueGetter(final DimensionSelector selector) if (selector.getValueCardinality() == 0) { return NULL_VALUE_GETTER; } else { - return new ValueGetter() - { - @Override - public String[] get() - { - final IndexedInts row = selector.getRow(); - final int size = row.size(); - if (size == 0) { - return NULL_VALUE; - } else { - String[] values = new String[size]; - for (int i = 0; i < size; ++i) { - values[i] = Strings.emptyToNull(selector.lookupName(row.get(i))); - } - return values; + return () -> { + final IndexedInts row = selector.getRow(); + final int size = row.size(); + if (size == 0) { + return NULL_VALUE; + } else { + String[] values = new String[size]; + for (int i = 0; i < size; ++i) { + values[i] = selector.lookupName(row.get(i)); } + return values; } }; } diff --git a/processing/src/main/java/io/druid/query/filter/ValueGetter.java b/processing/src/main/java/io/druid/query/filter/ValueGetter.java index 72000a4756da..c478ce971b37 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueGetter.java +++ b/processing/src/main/java/io/druid/query/filter/ValueGetter.java @@ -19,6 +19,8 @@ package io.druid.query.filter; +import javax.annotation.Nullable; + /** */ public interface ValueGetter @@ -27,5 +29,6 @@ public interface ValueGetter // converted to strings. We should also add functions // for these and modify ColumnComparisonFilter to handle // comparing Long and Float columns to eachother. + @Nullable String[] get(); } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index d6104f3018e3..dbcacfa6cc20 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -58,6 +58,7 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; @@ -532,19 +533,19 @@ private static int compareDims(List dimensions, Row lhs, Row rhs) for (DimensionSpec dimension : dimensions) { final int dimCompare; if (dimension.getOutputType() == ValueType.LONG) { - dimCompare = Long.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).longValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).longValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToLong(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToLong(rhs.getRaw(dimension.getOutputName())) ); } else if (dimension.getOutputType() == ValueType.FLOAT) { - dimCompare = Float.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).floatValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).floatValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToFloat(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToFloat(rhs.getRaw(dimension.getOutputName())) ); } else if (dimension.getOutputType() == ValueType.DOUBLE) { - dimCompare = Double.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).doubleValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).doubleValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToDouble(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToDouble(rhs.getRaw(dimension.getOutputName())) ); } else { dimCompare = ((Ordering) Comparators.naturalNullsFirst()).compare( diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 6db22e1efb19..bc4abfa8d480 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -20,12 +20,12 @@ package io.druid.query.groupby.epinephelinae; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import io.druid.collections.NonBlockingPool; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; @@ -124,8 +124,8 @@ public static Sequence process( final ResourceHolder bufferHolder = intermediateResultsBufferPool.take(); - final String fudgeTimestampString = Strings.emptyToNull( - query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, "") + final String fudgeTimestampString = NullHandling.emptyToNullIfNeeded( + query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, null) ); final DateTime fudgeTimestamp = fudgeTimestampString == null @@ -685,19 +685,16 @@ private static void convertRowTypesToOutputTypes(List dimensionSp (dimName, baseVal) -> { switch (outputType) { case STRING: - baseVal = baseVal == null ? "" : baseVal.toString(); + baseVal = DimensionHandlerUtils.convertObjectToString(baseVal); break; case LONG: baseVal = DimensionHandlerUtils.convertObjectToLong(baseVal); - baseVal = baseVal == null ? 0L : baseVal; break; case FLOAT: baseVal = DimensionHandlerUtils.convertObjectToFloat(baseVal); - baseVal = baseVal == null ? 0.f : baseVal; break; case DOUBLE: baseVal = DimensionHandlerUtils.convertObjectToDouble(baseVal); - baseVal = baseVal == null ? 0.d : baseVal; break; default: throw new IAE("Unsupported type: " + outputType); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index ad5f12b98fd6..7c0d95180c5a 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -31,6 +30,7 @@ import com.google.common.primitives.Longs; import com.google.common.util.concurrent.ListeningExecutorService; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.common.utils.IntArrayUtils; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; @@ -39,6 +39,7 @@ import io.druid.java.util.common.Pair; import io.druid.java.util.common.granularity.AllGranularity; import io.druid.java.util.common.guava.Accumulator; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.BaseQuery; import io.druid.query.ColumnSelectorPlus; import io.druid.query.aggregation.AggregatorFactory; @@ -438,7 +439,7 @@ public Row apply(Grouper.Entry entry) Object dimVal = entry.getKey().getKey()[i]; theMap.put( query.getDimensions().get(i - dimStart).getOutputName(), - dimVal instanceof String ? Strings.emptyToNull((String) dimVal) : dimVal + dimVal instanceof String ? NullHandling.emptyToNullIfNeeded((String) dimVal) : dimVal ); } @@ -527,18 +528,11 @@ private static class StringInputRawSupplierColumnSelectorStrategy @Override public Supplier makeInputRawSupplier(DimensionSelector selector) { - return new Supplier() - { - @Override - public Comparable get() - { - final String value; - IndexedInts index = selector.getRow(); - value = index.size() == 0 - ? "" - : selector.lookupName(index.get(0)); - return Strings.nullToEmpty(value); - } + return () -> { + IndexedInts index = selector.getRow(); + return index.size() == 0 + ? null + : selector.lookupName(index.get(0)); }; } } @@ -605,28 +599,19 @@ private static Function[] makeValueConvertFunctions( type = type == null ? ValueType.STRING : type; switch (type) { case STRING: - functions[i] = input -> input == null ? "" : input.toString(); + functions[i] = input -> DimensionHandlerUtils.convertObjectToString(input); break; case LONG: - functions[i] = input -> { - final Long val = DimensionHandlerUtils.convertObjectToLong(input); - return val == null ? 0L : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToLong(input); break; case FLOAT: - functions[i] = input -> { - final Float val = DimensionHandlerUtils.convertObjectToFloat(input); - return val == null ? 0.f : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToFloat(input); break; case DOUBLE: - functions[i] = input -> { - Double val = DimensionHandlerUtils.convertObjectToDouble(input); - return val == null ? 0.0 : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToDouble(input); break; default: throw new IAE("invalid type: [%s]", type); @@ -871,7 +856,10 @@ public int compare(Grouper.Entry entry1, Grouper.Entry private static int compareDimsInRows(RowBasedKey key1, RowBasedKey key2, int dimStart) { for (int i = dimStart; i < key1.getKey().length; i++) { - final int cmp = ((Comparable) key1.getKey()[i]).compareTo(key2.getKey()[i]); + final int cmp = Comparators.naturalNullsFirst().compare( + (Comparable) key1.getKey()[i], + (Comparable) key2.getKey()[i] + ); if (cmp != 0) { return cmp; } @@ -920,9 +908,10 @@ private static int compareDimsInRowsWithAggs( if (isNumericField.get(i) && comparator.equals(StringComparators.NUMERIC)) { // use natural comparison - cmp = lhs.compareTo(rhs); + cmp = Comparators.naturalNullsFirst().compare(lhs, rhs); } else { - cmp = comparator.compare(lhs.toString(), rhs.toString()); + cmp = comparator.compare(DimensionHandlerUtils.convertObjectToString(lhs), + DimensionHandlerUtils.convertObjectToString(rhs)); } if (cmp != 0) { @@ -936,7 +925,8 @@ private static int compareDimsInRowsWithAggs( static long estimateStringKeySize(String key) { - return (long) key.length() * Character.BYTES + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; + long length = key == null ? 0 : key.length(); + return length * Character.BYTES + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; } private static class RowBasedKeySerde implements Grouper.KeySerde @@ -1024,7 +1014,7 @@ private void initializeRankOfDictionaryIds() rankOfDictionaryIds = IntStream.range(0, dictionarySize).toArray(); IntArrays.quickSort( rankOfDictionaryIds, - (i1, i2) -> dictionary.get(i1).compareTo(dictionary.get(i2)) + (i1, i2) -> Comparators.naturalNullsFirst().compare(dictionary.get(i1), dictionary.get(i2)) ); IntArrayUtils.inverse(rankOfDictionaryIds); @@ -1581,14 +1571,30 @@ public int getKeyBufferValueSize() @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putLong((Long) key.getKey()[idx]); + Long aLong = (Long) key.getKey()[idx]; + if (aLong == null) { + keyBuffer.put((byte) 1); + keyBuffer.putLong(0L); + } else { + keyBuffer.put((byte) 0); + keyBuffer.putLong(aLong); + } return true; } + protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandling.defaultLongValue(); + } else { + return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); + } + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getLong(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override @@ -1606,7 +1612,8 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper FloatRowBasedKeySerdeHelper( int keyBufferPosition, boolean pushLimitDown, - @Nullable StringComparator stringComparator) + @Nullable StringComparator stringComparator + ) { this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { @@ -1632,14 +1639,30 @@ public int getKeyBufferValueSize() @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putFloat((Float) key.getKey()[idx]); + Float aFloat = (Float) key.getKey()[idx]; + if (aFloat == null) { + keyBuffer.put((byte) 1); + keyBuffer.putFloat(0F); + } else { + keyBuffer.put((byte) 0); + keyBuffer.putFloat(aFloat); + } return true; } + protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandling.defaultFloatValue(); + } else { + return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); + } + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getFloat(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override @@ -1684,14 +1707,30 @@ public int getKeyBufferValueSize() @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putDouble((Double) key.getKey()[idx]); + Double aDouble = (Double) key.getKey()[idx]; + if (aDouble == null) { + keyBuffer.put((byte) 1); + keyBuffer.putDouble(0.0D); + } else { + keyBuffer.put((byte) 0); + keyBuffer.putDouble(aDouble); + } return true; } + protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandling.defaultDoubleValue(); + } else { + return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); + } + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getDouble(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index db58c36e0579..f5bf10fbe6de 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.data.ArrayBasedIndexedInts; @@ -59,7 +60,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, value ); } else { - resultMap.put(selectorPlus.getOutputName(), ""); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultStringValue()); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index 38de1f7b9b97..cc249b84dd18 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -20,6 +20,7 @@ package io.druid.query.groupby.epinephelinae.column; import com.google.common.base.Preconditions; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; @@ -47,7 +48,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ((DimensionSelector) selectorPlus.getSelector()).lookupName(id) ); } else { - resultMap.put(selectorPlus.getOutputName(), ""); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultStringValue()); } } diff --git a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java index 2e382c4dd9cd..7f1127d92519 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java @@ -68,6 +68,10 @@ public void setAggregators(Map aggregators) @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return metricVal == null && value == null; + } return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) == 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java index 1c7ec437eecd..06fe975aea6d 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java @@ -68,6 +68,10 @@ public void setAggregators(Map aggregators) @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return false; + } return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) > 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java index 37d6268b866d..6ab9747639c2 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java @@ -66,6 +66,10 @@ public void setAggregators(Map aggregators) @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return false; + } return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) < 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java index 5ad75004b760..539fb7b39d63 100644 --- a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java +++ b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java @@ -228,6 +228,14 @@ protected BinaryFn createMergeFn(Query queryParam) // Fudge timestamp, maybe. final DateTime fudgeTimestamp = getUniversalTimestamp(query); + ImmutableMap.Builder context = ImmutableMap.builder(); + context.put("finalize", false); + context.put(GroupByQueryConfig.CTX_KEY_STRATEGY, GroupByStrategySelector.STRATEGY_V2); + if (fudgeTimestamp != null) { + context.put(CTX_KEY_FUDGE_TIMESTAMP, String.valueOf(fudgeTimestamp.getMillis())); + } + context.put(CTX_KEY_OUTERMOST, false); + context.put(GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown()); final GroupByQuery newQuery = new GroupByQuery( query.getDataSource(), @@ -243,14 +251,7 @@ protected BinaryFn createMergeFn(Query queryParam) query.getLimitSpec(), query.getContext() ).withOverriddenContext( - ImmutableMap.of( - "finalize", false, - GroupByQueryConfig.CTX_KEY_STRATEGY, GroupByStrategySelector.STRATEGY_V2, - CTX_KEY_FUDGE_TIMESTAMP, fudgeTimestamp == null ? "" : String.valueOf(fudgeTimestamp.getMillis()), - CTX_KEY_OUTERMOST, false, - // the having spec shouldn't be passed down, so we need to convey the existing limit push down status - GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown() - ) + context.build() ); Sequence rowSequence = Sequences.map( diff --git a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java index 810afa9d3d2e..f5fef048e94f 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java @@ -56,7 +56,9 @@ public LookupConfig( @JsonProperty("snapshotWorkingDir") String snapshotWorkingDir ) { + //CHECKSTYLE.OFF: Regexp this.snapshotWorkingDir = Strings.nullToEmpty(snapshotWorkingDir); + //CHECKSTYLE.ON: Regexp } public String getSnapshotWorkingDir() diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java index 0400c21ae1e6..1a487fab63d5 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java @@ -25,7 +25,6 @@ import io.druid.query.extraction.MapLookupExtractor; import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -45,7 +44,7 @@ public abstract class LookupExtractor * @return The lookup, or null key cannot have the lookup applied to it and should be treated as missing. */ @Nullable - public abstract String apply(@NotNull String key); + public abstract String apply(@Nullable String key); /** * @param keys set of keys to apply lookup for each element diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 6f7b7f7a0ce2..b10f6e0d0b06 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -21,10 +21,10 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Accumulator; @@ -216,8 +216,8 @@ private ColumnAnalysis analyzeStringColumn( } if (analyzingMinMax() && cardinality > 0) { - min = Strings.nullToEmpty(bitmapIndex.getValue(0)); - max = Strings.nullToEmpty(bitmapIndex.getValue(cardinality - 1)); + min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0)); + max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1)); } return new ColumnAnalysis( diff --git a/processing/src/main/java/io/druid/query/search/SearchHit.java b/processing/src/main/java/io/druid/query/search/SearchHit.java index 52e6692f989a..4031d0df3d88 100644 --- a/processing/src/main/java/io/druid/query/search/SearchHit.java +++ b/processing/src/main/java/io/druid/query/search/SearchHit.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.common.config.NullHandling; /** */ @@ -39,7 +40,7 @@ public SearchHit( ) { this.dimension = Preconditions.checkNotNull(dimension); - this.value = Preconditions.checkNotNull(value); + this.value = NullHandling.nullToEmptyIfNeeded(value); this.count = count; } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 7921d717b9a4..890d8c7ec328 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -20,7 +20,6 @@ package io.druid.query.search; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -131,7 +130,7 @@ public void updateSearchResultSet( for (int i = 0; i < vals.size(); ++i) { final String dimVal = selector.lookupName(vals.get(i)); if (searchQuerySpec.accept(dimVal)) { - set.addTo(new SearchHit(outputName, Strings.nullToEmpty(dimVal)), 1); + set.addTo(new SearchHit(outputName, dimVal), 1); if (set.size() >= limit) { return; } diff --git a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java index ec4ce2078546..987a86e0d685 100644 --- a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java @@ -20,7 +20,6 @@ package io.druid.query.search; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.collections.bitmap.BitmapFactory; @@ -261,7 +260,7 @@ public Object2IntRBTreeMap execute(int limit) extractionFn = IdentityExtractionFn.getInstance(); } for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + String dimVal = extractionFn.apply(bitmapIndex.getValue(i)); if (!searchQuerySpec.accept(dimVal)) { continue; } diff --git a/processing/src/main/java/io/druid/query/topn/TopNMapFn.java b/processing/src/main/java/io/druid/query/topn/TopNMapFn.java index 24d05fa37775..3d4807628ec9 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNMapFn.java +++ b/processing/src/main/java/io/druid/query/topn/TopNMapFn.java @@ -56,14 +56,10 @@ public static Function getValueTransformer(ValueType outputType) return longVal == null ? DimensionHandlerUtils.ZERO_LONG : longVal; }; - private static Function FLOAT_TRANSFORMER = input -> { - final Float floatVal = DimensionHandlerUtils.convertObjectToFloat(input); - return floatVal == null ? DimensionHandlerUtils.ZERO_FLOAT : floatVal; - }; - private static Function DOUBLE_TRANSFORMER = input -> { - final Double doubleValue = DimensionHandlerUtils.convertObjectToDouble(input); - return doubleValue == null ? DimensionHandlerUtils.ZERO_DOUBLE : doubleValue; - }; + private static Function FLOAT_TRANSFORMER = input -> DimensionHandlerUtils.convertObjectToFloat(input); + + private static Function DOUBLE_TRANSFORMER = input -> DimensionHandlerUtils.convertObjectToDouble( + input); private static final TopNColumnSelectorStrategyFactory STRATEGY_FACTORY = new TopNColumnSelectorStrategyFactory(); diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index bee78e103048..45f050f93616 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -225,7 +225,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) } final BitmapIndex bitmapIndex = column.getBitmapIndex(); - return bitmapIndex.getBitmap(bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(value))); + return bitmapIndex.getBitmap(bitmapIndex.getIndex(value)); } @Override diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index d46e8be45a6a..f00997027790 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -237,6 +237,15 @@ private static Colu return strategyFactory.makeColumnSelectorStrategy(capabilities, selector); } + @Nullable + public static String convertObjectToString(@Nullable Object valObj) + { + if (valObj == null) { + return null; + } + return valObj.toString(); + } + @Nullable public static Long convertObjectToLong(@Nullable Object valObj) { diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index a0a7f30e31f3..2093d1afe388 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -44,9 +44,7 @@ public Double processRowValsToUnsortedEncodedKeyComponent(Object dimValues, bool if (dimValues instanceof List) { throw new UnsupportedOperationException("Numeric columns do not support multivalue rows."); } - Double ret = DimensionHandlerUtils.convertObjectToDouble(dimValues, reportParseExceptions); - // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged - return ret == null ? DimensionHandlerUtils.ZERO_DOUBLE : ret; + return DimensionHandlerUtils.convertObjectToDouble(dimValues, reportParseExceptions); } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index 695535cdac5a..f349847038e4 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -45,9 +45,7 @@ public Float processRowValsToUnsortedEncodedKeyComponent(Object dimValues, boole throw new UnsupportedOperationException("Numeric columns do not support multivalue rows."); } - Float ret = DimensionHandlerUtils.convertObjectToFloat(dimValues, reportParseExceptions); - // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged - return ret == null ? DimensionHandlerUtils.ZERO_FLOAT : ret; + return DimensionHandlerUtils.convertObjectToFloat(dimValues, reportParseExceptions); } @Override diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index 187c3fe2d291..203000fbb5f1 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -45,9 +45,7 @@ public Long processRowValsToUnsortedEncodedKeyComponent(Object dimValues, boolea throw new UnsupportedOperationException("Numeric columns do not support multivalue rows."); } - Long ret = DimensionHandlerUtils.convertObjectToLong(dimValues, reportParseExceptions); - // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged - return ret == null ? DimensionHandlerUtils.ZERO_LONG : ret; + return DimensionHandlerUtils.convertObjectToLong(dimValues, reportParseExceptions); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index c9fc4262a1ee..c8115b804a39 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.Pair; import io.druid.query.BitmapResultFactory; import io.druid.query.extraction.ExtractionFn; @@ -148,7 +149,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasLowerBound()) { startIndex = 0; } else { - final int found = bitmapIndex.getIndex(boundDimFilter.getLower()); + final int found = bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getLower())); if (found >= 0) { startIndex = boundDimFilter.isLowerStrict() ? found + 1 : found; } else { @@ -159,7 +160,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasUpperBound()) { endIndex = bitmapIndex.getCardinality(); } else { - final int found = bitmapIndex.getIndex(boundDimFilter.getUpper()); + final int found = bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getUpper())); if (found >= 0) { endIndex = boundDimFilter.isUpperStrict() ? found : found + 1; } else { @@ -249,9 +250,10 @@ private boolean doesMatch(String input) { if (input == null) { return (!boundDimFilter.hasLowerBound() - || (boundDimFilter.getLower().isEmpty() && !boundDimFilter.isLowerStrict())) // lower bound allows null + || (NullHandling.isNullOrEquivalent(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) + // lower bound allows null && (!boundDimFilter.hasUpperBound() - || !boundDimFilter.getUpper().isEmpty() + || !NullHandling.isNullOrEquivalent(boundDimFilter.getUpper()) || !boundDimFilter.isUpperStrict()); // upper bound allows null } int lowerComparing = 1; diff --git a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java index 4acea0f29376..c13d9edd3aa0 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import io.druid.common.config.NullHandling; import io.druid.math.expr.Evals; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -108,7 +109,8 @@ public T getBitmapResult(final BitmapIndexSelector selector, final BitmapRes value -> expr.eval(identifierName -> { // There's only one binding, and it must be the single column, so it can safely be ignored in production. assert column.equals(identifierName); - return value; + // convert null to Empty before passing to expressions if needed. + return NullHandling.nullToEmptyIfNeeded(value); }).asBoolean() ); } diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index 684fd2f28fb5..dd5daa9281e8 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.common.config.NullHandling; import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.query.BitmapResultFactory; import io.druid.query.ColumnSelectorPlus; @@ -133,8 +132,7 @@ public static ValueMatcher makeValueMatcher( columnSelectorFactory ); - return selector.getColumnSelectorStrategy() - .makeValueMatcher(selector.getSelector(), NullHandling.emptyToNullIfNeeded(value)); + return selector.getColumnSelectorStrategy().makeValueMatcher(selector.getSelector(), value); } /** diff --git a/processing/src/main/java/io/druid/segment/filter/InFilter.java b/processing/src/main/java/io/druid/segment/filter/InFilter.java index 7e737616dadd..869e94e37ef7 100644 --- a/processing/src/main/java/io/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/InFilter.java @@ -20,7 +20,6 @@ package io.druid.segment.filter; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.BitmapResultFactory; @@ -165,9 +164,9 @@ private DruidPredicateFactory getPredicateFactory() public Predicate makeStringPredicate() { if (extractionFn != null) { - return input -> values.contains(Strings.nullToEmpty(extractionFn.apply(input))); + return input -> values.contains(extractionFn.apply(input)); } else { - return input -> values.contains(Strings.nullToEmpty(input)); + return input -> values.contains(input); } } diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index a5984f31b5c8..99b953c6270f 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -19,9 +19,9 @@ package io.druid.segment.filter; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.BitmapResultFactory; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; @@ -90,7 +90,10 @@ private Iterable getBitmapIterable(final BitmapIndexSelector se { if (isSimpleEquals()) { // Verify that dimension equals prefix. - return ImmutableList.of(selector.getBitmapIndex(dimension, likeMatcher.getPrefix())); + return ImmutableList.of(selector.getBitmapIndex( + dimension, + NullHandling.emptyToNullIfNeeded(likeMatcher.getPrefix()) + )); } else if (isSimplePrefix()) { // Verify that dimension startsWith prefix, and is accepted by likeMatcher.matchesSuffixOnly. final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension); @@ -140,16 +143,24 @@ private IntIterable getDimValueIndexIterableForPrefixMatch( final Indexed dimValues ) { - final String lower = Strings.nullToEmpty(likeMatcher.getPrefix()); - final String upper = Strings.nullToEmpty(likeMatcher.getPrefix()) + Character.MAX_VALUE; + + final String lower = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()); + final String upper = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE; + final int startIndex; // inclusive final int endIndex; // exclusive - final int lowerFound = bitmapIndex.getIndex(lower); - startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1); + if (lower == null) { + // For Null values + startIndex = bitmapIndex.getIndex(null); + endIndex = startIndex + 1; + } else { + final int lowerFound = bitmapIndex.getIndex(lower); + startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1); - final int upperFound = bitmapIndex.getIndex(upper); - endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1); + final int upperFound = bitmapIndex.getIndex(upper); + endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1); + } return new IntIterable() { diff --git a/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java index 72d20f9c2355..ea204ced0d11 100644 --- a/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java @@ -22,7 +22,6 @@ import com.google.common.base.Supplier; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; -import io.druid.common.config.NullHandling; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.GenericIndexed; @@ -78,7 +77,7 @@ public BitmapFactory getBitmapFactory() public int getIndex(String value) { // GenericIndexed.indexOf satisfies contract needed by BitmapIndex.indexOf - return dictionary.indexOf(NullHandling.emptyToNullIfNeeded(value)); + return dictionary.indexOf(value); } @Override diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index 8698b36e1f53..a17584bf16fe 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -21,13 +21,11 @@ import com.google.common.base.Function; import io.druid.guice.annotations.ExtensionPoint; -import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.ObjectStrategy; -import it.unimi.dsi.fastutil.bytes.ByteArrays; +import io.druid.segment.writeout.SegmentWriteOutMedium; -import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -82,9 +80,9 @@ public Function inputSizeFn() * * @return serialized intermediate representation of aggregate in byte[] */ - public byte[] toBytes(@Nullable Object val) + public byte[] toBytes(Object val) { - return val != null ? getObjectStrategy().toBytes(val) : ByteArrays.EMPTY_ARRAY; + return getObjectStrategy().toBytes(val); } /** From 51be891e13c155811e5356bddda5df9f988a6b73 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 27 Feb 2018 03:05:33 +0530 Subject: [PATCH 02/38] review comment - aggregator method naming, avoid using Pair --- .../java/io/druid/indexer/InputRowSerde.java | 43 ++-- .../io/druid/indexer/InputRowSerdeTest.java | 31 ++- .../query/aggregation/AggregatorUtil.java | 9 +- .../DoubleMaxAggregatorFactory.java | 31 +-- .../DoubleMinAggregatorFactory.java | 31 +-- .../DoubleSumAggregatorFactory.java | 29 +-- .../FloatMaxAggregatorFactory.java | 28 +-- .../FloatMinAggregatorFactory.java | 28 +-- .../FloatSumAggregatorFactory.java | 28 ++- .../aggregation/LongMaxAggregatorFactory.java | 26 +- .../aggregation/LongMinAggregatorFactory.java | 24 +- .../aggregation/LongSumAggregatorFactory.java | 27 ++- .../NullableAggregatorFactory.java | 22 +- .../SimpleDoubleAggregatorFactory.java | 4 +- .../SimpleFloatAggregatorFactory.java | 4 +- .../first/DoubleFirstAggregatorFactory.java | 101 ++++---- .../first/FloatFirstAggregatorFactory.java | 104 ++++---- .../first/LongFirstAggregatorFactory.java | 103 ++++---- .../last/DoubleLastAggregatorFactory.java | 103 ++++---- .../last/FloatLastAggregatorFactory.java | 103 ++++---- .../last/LongLastAggregatorFactory.java | 103 ++++---- .../epinephelinae/RowBasedGrouperHelper.java | 6 +- .../query/groupby/GroupByQueryRunnerTest.java | 2 + .../io/druid/segment/IndexMergerTestBase.java | 224 ++++++++++++------ 24 files changed, 645 insertions(+), 569 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index cd1dd531604a..dea993a83386 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -45,6 +45,7 @@ import io.druid.segment.serde.ComplexMetrics; import org.apache.hadoop.io.WritableUtils; +import javax.annotation.Nullable; import java.io.DataInput; import java.io.IOException; import java.util.List; @@ -55,6 +56,8 @@ public class InputRowSerde { private static final Logger log = new Logger(InputRowSerde.class); + private static final byte NULL_BYTE = (byte) 1; + private static final byte NON_NULL_BYTE = (byte) 0; private static final IndexSerdeTypeHelper STRING_HELPER = new StringIndexSerdeTypeHelper(); private static final IndexSerdeTypeHelper LONG_HELPER = new LongIndexSerdeTypeHelper(); @@ -271,18 +274,22 @@ public InputRow get() } String t = aggFactory.getTypeName(); - - if (t.equals("float")) { - out.writeFloat(agg.getFloat()); - } else if (t.equals("long")) { - WritableUtils.writeVLong(out, agg.getLong()); - } else if (t.equals("double")) { - out.writeDouble(agg.getDouble()); + if (agg.isNull()) { + out.writeByte(NULL_BYTE); } else { - //its a complex metric - Object val = agg.get(); - ComplexMetricSerde serde = getComplexMetricSerde(t); - writeBytes(serde.toBytes(val), out); + out.writeByte(NON_NULL_BYTE); + if (t.equals("float")) { + out.writeFloat(agg.getFloat()); + } else if (t.equals("long")) { + WritableUtils.writeVLong(out, agg.getLong()); + } else if (t.equals("double")) { + out.writeDouble(agg.getDouble()); + } else { + //its a complex metric + Object val = agg.get(); + ComplexMetricSerde serde = getComplexMetricSerde(t); + writeBytes(serde.toBytes(val), out); + } } } } @@ -294,10 +301,13 @@ public InputRow get() } } - private static void writeBytes(byte[] value, ByteArrayDataOutput out) throws IOException + private static void writeBytes(@Nullable byte[] value, ByteArrayDataOutput out) throws IOException { - WritableUtils.writeVInt(out, value.length); - out.write(value, 0, value.length); + int length = value == null ? -1 : value.length; + WritableUtils.writeVInt(out, length); + if (value != null) { + out.write(value, 0, value.length); + } } private static void writeString(String value, ByteArrayDataOutput out) throws IOException @@ -391,6 +401,11 @@ public static final InputRow fromBytes( for (int i = 0; i < metricSize; i++) { String metric = readString(in); String type = getType(metric, aggs, i); + byte metricNullability = in.readByte(); + if (metricNullability == NULL_BYTE) { + // metric value is null. + continue; + } if (type.equals("float")) { event.put(metric, in.readFloat()); } else if (type.equals("long")) { diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index 71609e42dd32..2e485f5224e6 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -84,6 +84,7 @@ public void testSerde() { // Prepare the mocks & set close() call count expectation to 1 final Aggregator mockedAggregator = EasyMock.createMock(DoubleSumAggregator.class); + EasyMock.expect(mockedAggregator.isNull()).andReturn(false).times(1); EasyMock.expect(mockedAggregator.getDouble()).andReturn(0d).times(1); mockedAggregator.aggregate(); EasyMock.expectLastCall().times(1); @@ -91,6 +92,26 @@ public void testSerde() EasyMock.expectLastCall().times(1); EasyMock.replay(mockedAggregator); + final Aggregator mockedNullAggregator = EasyMock.createMock(DoubleSumAggregator.class); + EasyMock.expect(mockedNullAggregator.isNull()).andReturn(true).times(1); + mockedNullAggregator.aggregate(); + EasyMock.expectLastCall().times(1); + mockedNullAggregator.close(); + EasyMock.expectLastCall().times(1); + EasyMock.replay(mockedNullAggregator); + + final AggregatorFactory mockedAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); + EasyMock.expect(mockedAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedAggregator); + EasyMock.expect(mockedAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + EasyMock.expect(mockedAggregatorFactory.getName()).andReturn("mockedAggregator").anyTimes(); + + final AggregatorFactory mockedNullAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); + EasyMock.expect(mockedNullAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedNullAggregator); + EasyMock.expect(mockedNullAggregatorFactory.getName()).andReturn("mockedNullAggregator").anyTimes(); + EasyMock.expect(mockedNullAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + + EasyMock.replay(mockedAggregatorFactory, mockedNullAggregatorFactory); + InputRow in = new MapBasedInputRow( timestamp, dims, @@ -103,13 +124,8 @@ public void testSerde() new LongSumAggregatorFactory("m2out", "m2"), new HyperUniquesAggregatorFactory("m3out", "m3"), new LongSumAggregatorFactory("unparseable", "m3"), // Unparseable from String to Long - new DoubleSumAggregatorFactory("mockedAggregator", "m4") { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - return mockedAggregator; - } - } + mockedAggregatorFactory, + mockedNullAggregatorFactory }; DimensionsSpec dimensionsSpec = new DimensionsSpec( @@ -143,6 +159,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) Assert.assertEquals(0L, out.getMetric("unparseable")); EasyMock.verify(mockedAggregator); + EasyMock.verify(mockedNullAggregator); } @Test diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index f1b2438fe02b..fe28ce163ffe 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -27,9 +27,6 @@ import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DoubleColumnSelector; @@ -137,7 +134,7 @@ public static Pair, List> condensedAggre return new Pair(condensedAggs, condensedPostAggs); } - public static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( + public static ColumnValueSelector makeColumnValueSelectorWithFloatDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, final String fieldName, @@ -178,7 +175,7 @@ public boolean isNull() throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } - public static BaseLongColumnValueSelector makeColumnValueSelectorWithLongDefault( + public static ColumnValueSelector makeColumnValueSelectorWithLongDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, final String fieldName, @@ -219,7 +216,7 @@ public boolean isNull() throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } - public static BaseDoubleColumnValueSelector makeColumnValueSelectorWithDoubleDefault( + public static ColumnValueSelector makeColumnValueSelectorWithDoubleDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, final String fieldName, diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index f23957dd2106..49cda751c7a1 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -22,12 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -56,29 +54,26 @@ public DoubleMaxAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + return getDoubleColumnSelector( metricFactory, Double.NEGATIVE_INFINITY ); - return Pair.of( - new DoubleMaxAggregator(doubleColumnSelector), - doubleColumnSelector - ); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( - metricFactory, - Double.NEGATIVE_INFINITY - ); - return Pair.of( - new DoubleMaxBufferAggregator(doubleColumnSelector), - doubleColumnSelector - ); + return new DoubleMaxAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new DoubleMaxBufferAggregator(selector); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 7869bb1f2ce3..46ede137418a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -22,12 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -55,29 +53,26 @@ public DoubleMinAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + return getDoubleColumnSelector( metricFactory, Double.POSITIVE_INFINITY ); - return Pair.of( - new DoubleMinAggregator(doubleColumnSelector), - doubleColumnSelector - ); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( - metricFactory, - Double.POSITIVE_INFINITY - ); - return Pair.of( - new DoubleMinBufferAggregator(doubleColumnSelector), - doubleColumnSelector - ); + return new DoubleMinAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new DoubleMinBufferAggregator(selector); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 313a2676fdba..bc5735c825e1 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -22,12 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -56,23 +54,26 @@ public DoubleSumAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); - return Pair.of( - new DoubleSumAggregator(doubleColumnSelector), - doubleColumnSelector + return getDoubleColumnSelector( + metricFactory, + 0.0d ); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); - return Pair.of( - new DoubleSumBufferAggregator(doubleColumnSelector), - doubleColumnSelector - ); + return new DoubleSumAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new DoubleSumBufferAggregator(selector); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index 589c1def93e7..8ae3c647bf35 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -22,12 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -55,26 +53,26 @@ public FloatMaxAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + return makeColumnValueSelectorWithFloatDefault( metricFactory, Float.NEGATIVE_INFINITY ); - return Pair.of(new FloatMaxAggregator(floatColumnSelector), floatColumnSelector); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( - metricFactory, - Float.NEGATIVE_INFINITY - ); - return Pair.of( - new FloatMaxBufferAggregator(floatColumnSelector), - floatColumnSelector - ); + return new FloatMaxAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new FloatMaxBufferAggregator(selector); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index f97d7c5ba29f..8b3b3334e25b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -22,12 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -55,26 +53,26 @@ public FloatMinAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + return makeColumnValueSelectorWithFloatDefault( metricFactory, Float.POSITIVE_INFINITY ); - return Pair.of(new FloatMinAggregator(floatColumnSelector), floatColumnSelector); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( - metricFactory, - Float.POSITIVE_INFINITY - ); - return Pair.of( - new FloatMinBufferAggregator(floatColumnSelector), - floatColumnSelector - ); + return new FloatMinAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new FloatMinBufferAggregator(selector); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index b6c7725cf970..a687ea653181 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -22,12 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -55,20 +53,26 @@ public FloatSumAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); - return Pair.of(new FloatSumAggregator(floatColumnSelector), floatColumnSelector); + return makeColumnValueSelectorWithFloatDefault( + metricFactory, + 0.0f + ); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); - return Pair.of( - new FloatSumBufferAggregator(floatColumnSelector), - floatColumnSelector - ); + return new FloatSumAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new FloatSumBufferAggregator(selector); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index 5b63da651935..c78365cdaa5f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -23,12 +23,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; @@ -75,23 +72,26 @@ public LongMaxAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return Pair.of(new LongMaxAggregator(longColumnSelector), longColumnSelector); + return getLongColumnSelector(metricFactory); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return Pair.of( - new LongMaxBufferAggregator(longColumnSelector), - longColumnSelector - ); + return new LongMaxAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new LongMaxBufferAggregator(selector); } - private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) + private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) { return AggregatorUtil.makeColumnValueSelectorWithLongDefault( metricFactory, diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index a1d556f8d349..7589065ce887 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -23,12 +23,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; @@ -76,23 +73,24 @@ public LongMinAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return Pair.of(new LongMinAggregator(longColumnSelector), longColumnSelector); + return getLongColumnSelector(metricFactory); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return Pair.of( - new LongMinBufferAggregator(longColumnSelector), - longColumnSelector - ); + return new LongMinAggregator(selector); + } + + @Override + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + { + return new LongMaxBufferAggregator(selector); } - private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) + private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) { return AggregatorUtil.makeColumnValueSelectorWithLongDefault( metricFactory, diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 44d0673b2a77..46f1b8b29d8b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -23,13 +23,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -74,23 +72,26 @@ public LongSumAggregatorFactory(String name, String fieldName) } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return Pair.of(new LongSumAggregator(longColumnSelector), longColumnSelector); + return getLongColumnSelector(metricFactory); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); - return Pair.of( - new LongSumBufferAggregator(longColumnSelector), - longColumnSelector - ); + return new LongSumAggregator(selector); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new LongSumBufferAggregator(selector); } - private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) + private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) { return AggregatorUtil.makeColumnValueSelectorWithLongDefault( metricFactory, diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 8999266181d7..a077ccaf7443 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -21,9 +21,8 @@ import io.druid.common.config.NullHandling; -import io.druid.java.util.common.Pair; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; /** * abstract class with functionality to wrap aggregator/bufferAggregator/combiner to make them Nullable. @@ -35,23 +34,24 @@ public abstract class NullableAggregatorFactory extends AggregatorFactory @Override public final Aggregator factorize(ColumnSelectorFactory metricFactory) { - Pair pair = factorize2( - metricFactory); - return NullHandling.replaceWithDefault() ? pair.lhs : new NullableAggregator(pair.lhs, pair.rhs); + ColumnValueSelector selector = selector(metricFactory); + Aggregator aggregator = factorize(metricFactory, selector); + return NullHandling.replaceWithDefault() ? aggregator : new NullableAggregator(aggregator, selector); } - protected abstract Pair factorize2(ColumnSelectorFactory metricfactory); + protected abstract ColumnValueSelector selector(ColumnSelectorFactory metricFactory); + + protected abstract Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); @Override public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - Pair pair = factorizeBuffered2( - metricFactory); - return NullHandling.replaceWithDefault() ? pair.lhs : new NullableBufferAggregator(pair.lhs, pair.rhs); + ColumnValueSelector selector = selector(metricFactory); + BufferAggregator aggregator = factorizeBuffered(metricFactory, selector); + return NullHandling.replaceWithDefault() ? aggregator : new NullableBufferAggregator(aggregator, selector); } - protected abstract Pair factorizeBuffered2(ColumnSelectorFactory metricfactory); - + protected abstract BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); @Override public final AggregateCombiner makeAggregateCombiner() diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 6a809a441695..e99f3ace70c6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -24,8 +24,8 @@ import com.google.common.base.Preconditions; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -61,7 +61,7 @@ public SimpleDoubleAggregatorFactory( ); } - protected BaseDoubleColumnValueSelector getDoubleColumnSelector(ColumnSelectorFactory metricFactory, double nullValue) + protected ColumnValueSelector getDoubleColumnSelector(ColumnSelectorFactory metricFactory, double nullValue) { return AggregatorUtil.makeColumnValueSelectorWithDoubleDefault( metricFactory, diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index cdc797658577..be1752ca582b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -23,8 +23,8 @@ import com.google.common.base.Preconditions; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; -import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.util.Collections; @@ -57,7 +57,7 @@ public SimpleFloatAggregatorFactory( ); } - BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( + ColumnValueSelector makeColumnValueSelectorWithFloatDefault( ColumnSelectorFactory metricFactory, float nullValue ) diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 0fc65fcb5e4d..3330b54a0852 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -25,7 +25,6 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.SerializablePair; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -35,8 +34,6 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; @@ -80,26 +77,26 @@ public DoubleFirstAggregatorFactory( } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new DoubleFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector + return metricFactory.makeColumnValueSelector(fieldName); + } + + @Override + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + { + return new DoubleFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector ); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new DoubleFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector + return new DoubleFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector ); } @@ -134,48 +131,44 @@ public AggregatorFactory getCombiningFactory() return new DoubleFirstAggregatorFactory(name, name) { @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new DoubleFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }, selector); + return new DoubleFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }; } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new DoubleFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + return new DoubleFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } }; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 775ec3a49eb0..9ec538b306d9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -25,7 +25,6 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.SerializablePair; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -35,7 +34,6 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; @@ -77,25 +75,29 @@ public FloatFirstAggregatorFactory( } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new FloatFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector); + return metricFactory.makeColumnValueSelector(fieldName); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new FloatFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - columnValueSelector - ), columnValueSelector); + return new FloatFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new FloatFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); } @Override @@ -129,48 +131,44 @@ public AggregatorFactory getCombiningFactory() return new FloatFirstAggregatorFactory(name, name) { @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new FloatFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }, selector); + return new FloatFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }; } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new FloatFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + return new FloatFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } }; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index b6e7634d40fe..e4d2e958a94d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; import io.druid.collections.SerializablePair; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -34,8 +33,6 @@ import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; @@ -71,25 +68,29 @@ public LongFirstAggregatorFactory( } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new LongFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return metricFactory.makeColumnValueSelector(fieldName); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new LongFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return new LongFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new LongFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); } @Override @@ -123,48 +124,44 @@ public AggregatorFactory getCombiningFactory() return new LongFirstAggregatorFactory(name, name) { @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new LongFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }, selector); + return new LongFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }; } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new LongFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } + return new LongFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } }; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 09756e2c5b0f..90a3e800799f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import io.druid.collections.SerializablePair; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -35,8 +34,6 @@ import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseDoubleColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; @@ -70,25 +67,29 @@ public DoubleLastAggregatorFactory( } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new DoubleLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - doubleColumnSelector - ), doubleColumnSelector); + return metricFactory.makeColumnValueSelector(fieldName); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new DoubleLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - doubleColumnSelector - ), doubleColumnSelector); + return new DoubleLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new DoubleLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); } @Override @@ -122,48 +123,44 @@ public AggregatorFactory getCombiningFactory() return new DoubleLastAggregatorFactory(name, name) { @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new DoubleLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }, selector); + return new DoubleLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }; } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new DoubleLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Long.BYTES, pair.rhs); - } - } + return new DoubleLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Double.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } }; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 5f28ac48dc0a..34d40eecc73e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import io.druid.collections.SerializablePair; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -35,8 +34,6 @@ import io.druid.query.aggregation.first.FloatFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseFloatColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; @@ -68,25 +65,29 @@ public FloatLastAggregatorFactory( } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new FloatLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - floatColumnSelector - ), floatColumnSelector); + return metricFactory.makeColumnValueSelector(fieldName); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new FloatLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - floatColumnSelector - ), floatColumnSelector); + return new FloatLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new FloatLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); } @Override @@ -120,48 +121,44 @@ public AggregatorFactory getCombiningFactory() return new FloatLastAggregatorFactory(name, name) { @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new FloatLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }, selector); + return new FloatLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }; } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new FloatLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Long.BYTES, pair.rhs); - } - } + return new FloatLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Long.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } }; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 6af3336d8c1a..3cc6455b7591 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import io.druid.collections.SerializablePair; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; @@ -35,8 +34,6 @@ import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseLongColumnValueSelector; -import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; @@ -67,25 +64,29 @@ public LongLastAggregatorFactory( } @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new LongLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return metricFactory.makeColumnValueSelector(fieldName); } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); - return Pair.of( - new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - longColumnSelector - ), longColumnSelector); + return new LongLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); + } + + @Override + protected BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, ColumnValueSelector selector + ) + { + return new LongLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + selector + ); } @Override @@ -119,48 +120,44 @@ public AggregatorFactory getCombiningFactory() return new LongLastAggregatorFactory(name, name) { @Override - public Pair factorize2(ColumnSelectorFactory metricFactory) + public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new LongLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }, selector); + return new LongLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }; } @Override - public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return Pair.of( - new LongLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } + return new LongLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }, selector); + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } }; } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 7c0d95180c5a..f4a1127c823e 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -1565,7 +1565,7 @@ private class LongRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Long.BYTES; + return Long.BYTES + Byte.BYTES; } @Override @@ -1633,7 +1633,7 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Float.BYTES; + return Float.BYTES + Byte.BYTES; } @Override @@ -1701,7 +1701,7 @@ private class DoubleRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Double.BYTES; + return Double.BYTES + Byte.BYTES; } @Override diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 0d0615393b77..55e934642c7c 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -8205,6 +8205,7 @@ public void testGroupByNumericStringsAsNumericWithDecoration() ) ) .setGranularity(QueryRunnerTestHelper.allGran) + .addOrderByColumn("ql") .build(); // "entertainment" rows are excluded by the decorated specs, they become empty rows @@ -8224,6 +8225,7 @@ public void testGroupByNumericStringsAsNumericWithDecoration() ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + System.out.println(results); TestHelper.assertExpectedObjects(expectedResults, results, ""); } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 93254f9fd80b..87ebfaffcb06 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -27,6 +27,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionSchema; @@ -236,11 +237,11 @@ public void testPersistWithDifferentDims() throws Exception Assert.assertArrayEquals(new int[][]{{0}, {1}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{1}, {0}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim1", "3")); - checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", "")); + checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "2")); } @@ -905,17 +906,17 @@ public void testNonLexicographicDimOrderMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {1}, {1}}, boatList.get(2).getDims()); Assert.assertArrayEquals(new Object[]{3L}, boatList.get(2).getMetrics()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d3", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "30000")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "40000")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d3", "50000")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d1", "100")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d1", "200")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d1", "300")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d2", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d2", null)); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "2000")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d2", "3000")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d2", "4000")); @@ -1022,16 +1023,16 @@ public void testMergeWithDimensionsList() throws Exception Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dimC").hasBitmapIndexes()); if (useBitmapIndexes) { - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2")); } - checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dimB", null)); } @@ -1126,11 +1127,11 @@ public void testDisjointDimMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList.get(4).getDims()); Assert.assertArrayEquals(new Object[]{1L}, boatList.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(3, 4), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(3, 4), adapter.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "3")); @@ -1150,14 +1151,14 @@ public void testDisjointDimMerge() throws Exception // dimA always has bitmap indexes, since it has them in indexA (it comes in through discovery). Assert.assertTrue(adapter2.getCapabilities("dimA").hasBitmapIndexes()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(3), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter2.getBitmapIndex("dimA", "2")); // dimB may or may not have bitmap indexes, since it comes in through explicit definition in indexB2. Assert.assertEquals(useBitmapIndexes, adapter2.getCapabilities("dimB").hasBitmapIndexes()); if (useBitmapIndexes) { - checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimB", "3")); @@ -1265,40 +1266,60 @@ public void testJointDimMerge() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); - Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {1}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {2}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {1}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(1, 2, 3), adapter.getBitmapIndex("d1", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); + } - checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "210")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d5", "520")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d7", "720")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d8", "810")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d8", "820")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d9", "911")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d9", "920")); @@ -1410,40 +1431,64 @@ public void testNoRollupMergeWithoutDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + if (NullHandling.replaceWithDefault()) { + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + + checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + + } else { + // NULL and EMPTY Strings are considered different + Assert.assertArrayEquals(new int[][]{{1}, {1}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {2}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {1}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + + } + - checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "210")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d5", "520")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d7", "720")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", "")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d8", "810")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d8", "820")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d9", "911")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d9", "920")); @@ -1475,7 +1520,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1484,7 +1529,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1499,7 +1544,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1508,7 +1553,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 4, Arrays.asList("d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d5", "", "d6", "621", "d7", "", "d8", "821", "d9", "921" + "d6", "621", "d7", "", "d8", "821", "d9", "921" ) ) ); @@ -1554,26 +1599,41 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d3", "d6", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); - Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {1}, {1}}, boatList.get(3).getDims()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {1}, {1}}, boatList.get(3).getDims()); + } else { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {1}, {1}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3), adapter.getBitmapIndex("d7", "")); + } + - checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", "")); + checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d3", "310")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d9", "921")); } @@ -1606,7 +1666,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "1", "dimA", "") + ImmutableMap.of("dimB", "1") ) ); @@ -1614,7 +1674,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "", "dimA", "1") + ImmutableMap.of("dimA", "1") ) ); @@ -1730,11 +1790,11 @@ public void testMergeWithSupersetOrdering() throws Exception Assert.assertArrayEquals(new int[][]{{3}, {0}}, boatList.get(4).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimB", "3")); @@ -1770,16 +1830,16 @@ public void testMergeWithSupersetOrdering() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}, {0}}, boatList2.get(11).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList2.get(11).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(6), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(7, 11), adapter2.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(3, 8), adapter2.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(4, 9), adapter2.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(5, 10), adapter2.getBitmapIndex("dimB", "3")); - checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", null)); checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimC", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimC", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimC", "3")); @@ -2199,13 +2259,13 @@ public void testPersistNullColumnSkipping() throws Exception }); index1.add(new MapBasedInputRow( 1L, - Lists.newArrayList("d1", "d2"), + Lists.newArrayList("d1", "d2", "d3"), ImmutableMap.of("d1", "a", "d2", "", "A", 1) )); index1.add(new MapBasedInputRow( 1L, - Lists.newArrayList("d1", "d2"), + Lists.newArrayList("d1", "d2", "d3"), ImmutableMap.of("d1", "b", "d2", "", "A", 1) )); @@ -2220,14 +2280,30 @@ public void testPersistNullColumnSkipping() throws Exception ) ) ); - List expectedColumnNames = Arrays.asList("A", "d1"); + List expectedColumnNames = NullHandling.replaceWithDefault() + ? Arrays.asList("A", "d1") + : Arrays.asList("A", "d1", "d2"); List actualColumnNames = Lists.newArrayList(index.getColumnNames()); Collections.sort(expectedColumnNames); Collections.sort(actualColumnNames); Assert.assertEquals(expectedColumnNames, actualColumnNames); SmooshedFileMapper sfm = closer.closeLater(SmooshedFileMapper.load(tempDir)); - List expectedFilenames = Arrays.asList("A", "__time", "d1", "index.drd", "metadata.drd"); + List expectedFilenames = NullHandling.replaceWithDefault() ? Arrays.asList( + "A", + "__time", + "d1", + "index.drd", + "metadata.drd" + ) + : Arrays.asList( + "A", + "__time", + "d1", + "d2", + "index.drd", + "metadata.drd" + ); List actualFilenames = new ArrayList<>(sfm.getInternalFilenames()); Collections.sort(expectedFilenames); Collections.sort(actualFilenames); @@ -2416,7 +2492,7 @@ public void testMultiValueHandling() throws Exception Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); if (useBitmapIndexes) { - checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); @@ -2444,7 +2520,7 @@ public void testMultiValueHandling() throws Exception Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); if (useBitmapIndexes) { - checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "x")); @@ -2473,7 +2549,7 @@ public void testMultiValueHandling() throws Exception Assert.assertEquals(useBitmapIndexes, adapter.getCapabilities("dim2").hasBitmapIndexes()); if (useBitmapIndexes) { - checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList<>(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); From f1f655ecf704f5c14f3eeed0114405da993932df Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 27 Feb 2018 04:08:38 +0530 Subject: [PATCH 03/38] More fixes fix more tests more test fixes fix more tests more test fixes Fix more tests --- .travis.yml | 9 + .../main/java/io/druid/math/expr/Evals.java | 4 +- .../main/java/io/druid/math/expr/Expr.java | 9 +- .../java/io/druid/math/expr/ExprEval.java | 59 +++-- .../java/io/druid/math/expr/Function.java | 6 +- .../java/io/druid/math/expr/EvalTest.java | 7 +- .../query/aggregation/AggregatorUtil.java | 6 +- .../aggregation/LongMinAggregatorFactory.java | 2 +- .../NullableAggregatorFactory.java | 20 +- .../query/lookup/LookupExtractionFn.java | 6 +- .../io/druid/query/SchemaEvolutionTest.java | 18 +- .../aggregation/DoubleMaxAggregationTest.java | 10 +- .../aggregation/DoubleMinAggregationTest.java | 10 +- .../aggregation/FilteredAggregatorTest.java | 7 +- .../aggregation/LongMaxAggregationTest.java | 10 +- .../aggregation/LongMinAggregationTest.java | 10 +- .../CardinalityAggregatorTest.java | 15 +- .../first/DoubleFirstAggregationTest.java | 14 +- .../first/FloatFirstAggregationTest.java | 14 +- .../first/LongFirstAggregationTest.java | 14 +- .../last/DoubleLastAggregationTest.java | 14 +- .../last/FloatLastAggregationTest.java | 14 +- .../last/LongLastAggregationTest.java | 14 +- .../extraction/FunctionalExtractionTest.java | 25 +- .../JavaScriptExtractionFnTest.java | 7 +- .../extraction/LowerExtractionFnTest.java | 3 +- .../MatchingDimExtractionFnTest.java | 3 +- .../extraction/RegexDimExtractionFnTest.java | 17 +- .../extraction/StrlenExtractionFnTest.java | 3 +- .../extraction/TimeDimExtractionFnTest.java | 7 +- .../extraction/UpperExtractionFnTest.java | 3 +- .../query/groupby/GroupByQueryRunnerTest.java | 242 ++++++++++++++---- .../epinephelinae/BufferHashGrouperTest.java | 9 +- .../LimitedBufferHashGrouperTest.java | 106 +++++--- .../StreamingMergeSortedGrouperTest.java | 16 +- .../LookupExtractionFnExpectationsTest.java | 7 +- .../query/search/SearchQueryRunnerTest.java | 3 +- .../timeseries/TimeseriesQueryRunnerTest.java | 157 +++++++----- .../druid/query/topn/TopNQueryRunnerTest.java | 20 +- .../ConstantDimensionSelectorTest.java | 3 +- .../druid/segment/SchemalessTestFullTest.java | 76 +++--- .../segment/SchemalessTestSimpleTest.java | 3 +- .../java/io/druid/segment/TestHelper.java | 30 ++- .../druid/segment/filter/BoundFilterTest.java | 225 ++++++++++++---- .../filter/ColumnComparisonFilterTest.java | 31 ++- .../segment/filter/ExpressionFilterTest.java | 31 ++- .../segment/filter/FilterPartitionTest.java | 185 +++++++++---- .../io/druid/segment/filter/InFilterTest.java | 135 +++++++--- .../segment/filter/JavaScriptFilterTest.java | 15 +- .../druid/segment/filter/LikeFilterTest.java | 16 +- .../druid/segment/filter/RegexFilterTest.java | 24 +- .../segment/filter/SearchQueryFilterTest.java | 53 +++- .../segment/filter/SelectorFilterTest.java | 72 +++++- .../virtual/ExpressionVirtualColumnTest.java | 24 +- 54 files changed, 1331 insertions(+), 512 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fb64a7bf9af..285b3895bb29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,15 @@ matrix: - unset _JAVA_OPTIONS script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl processing + # processing module tests with SQL Compatibility enabled + - sudo: false + env: + - NAME="processing module test with SQL Compatibility" + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.generic.useDefaultValueForNull=false -pl processing + # server module test - sudo: false env: diff --git a/common/src/main/java/io/druid/math/expr/Evals.java b/common/src/main/java/io/druid/math/expr/Evals.java index 2b90b58cddf6..51d4b8b35c1e 100644 --- a/common/src/main/java/io/druid/math/expr/Evals.java +++ b/common/src/main/java/io/druid/math/expr/Evals.java @@ -19,7 +19,7 @@ package io.druid.math.expr; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.logger.Logger; import java.util.Arrays; @@ -83,6 +83,6 @@ public static boolean asBoolean(double x) public static boolean asBoolean(String x) { - return !Strings.isNullOrEmpty(x) && Boolean.valueOf(x); + return !NullHandling.isNullOrEquivalent(x) && Boolean.valueOf(x); } } diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 0b1deb188740..ecde03215d3c 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -365,15 +365,18 @@ public ExprEval eval(ObjectBinding bindings) // Result of any Binary expressions is null if any of the argument is null. // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec. - if (NullHandling.sqlCompatible() && (leftVal.isNull() || rightVal.isNull())) { - return ExprEval.of(null); - } if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) { return evalString(leftVal.asString(), rightVal.asString()); } else if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) { + if (NullHandling.sqlCompatible() && (leftVal.isNull() || rightVal.isNull())) { + return ExprEval.of(null); + } return ExprEval.of(evalLong(leftVal.asLong(), rightVal.asLong())); } else { + if (NullHandling.sqlCompatible() && (leftVal.isNull() || rightVal.isNull())) { + return ExprEval.of(null); + } return ExprEval.of(evalDouble(leftVal.asDouble(), rightVal.asDouble())); } } diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index c5f0f6a1c379..a1e4306f4fd1 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -21,7 +21,6 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.google.common.primitives.Ints; import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.IAE; @@ -99,6 +98,9 @@ public Object value() return value; } + /** + * returns true if numeric primitive value for this ExprEval is null, otherwise false. + */ public boolean isNull() { return value == null; @@ -175,7 +177,7 @@ public final ExprEval castTo(ExprType castTo) case DOUBLE: return this; case LONG: - return ExprEval.of(asLong()); + return ExprEval.of(value == null ? null : asLong()); case STRING: return ExprEval.of(asString()); } @@ -213,7 +215,7 @@ public final ExprEval castTo(ExprType castTo) { switch (castTo) { case DOUBLE: - return ExprEval.of(asDouble()); + return ExprEval.of(value == null ? null : asDouble()); case LONG: return this; case STRING: @@ -245,36 +247,55 @@ public final ExprType type() @Override public final int asInt() { - if (value == null) { + Number number = asNumber(); + if (number == null) { assert NullHandling.replaceWithDefault(); return 0; } - - final Integer theInt = Ints.tryParse(value); - assert NullHandling.replaceWithDefault() || theInt != null; - return theInt == null ? 0 : theInt; + return number.intValue(); } @Override public final long asLong() { - // GuavaUtils.tryParseLong handles nulls, no need for special null handling here. - final Long theLong = GuavaUtils.tryParseLong(value); - assert NullHandling.replaceWithDefault() || theLong != null; - return theLong == null ? 0L : theLong; + Number number = asNumber(); + if (number == null) { + assert NullHandling.replaceWithDefault(); + return 0L; + } + return number.longValue(); } @Override public final double asDouble() { - if (value == null) { + Number number = asNumber(); + if (number == null) { assert NullHandling.replaceWithDefault(); - return 0.0; + return 0.0d; + } + return number.doubleValue(); + } + + @Nullable + private Number asNumber() + { + if (value == null) { + return null; + } + Long v = GuavaUtils.tryParseLong(value); + // Do NOT use ternary operator here, because it makes Java to convert Long to Double + if (v != null) { + return v; + } else { + return Doubles.tryParse(value); } + } - final Double theDouble = Doubles.tryParse(value); - assert NullHandling.replaceWithDefault() || theDouble != null; - return theDouble == null ? 0.0 : theDouble; + @Override + public boolean isNull() + { + return !NullHandling.replaceWithDefault() && asNumber() == null; } @Override @@ -288,9 +309,9 @@ public final ExprEval castTo(ExprType castTo) { switch (castTo) { case DOUBLE: - return ExprEval.of(asDouble()); + return ExprEval.ofDouble(asNumber()); case LONG: - return ExprEval.of(asLong()); + return ExprEval.ofLong(asNumber()); case STRING: return this; } diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 42b7902ba4bd..672dae0315d3 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -880,7 +880,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) throw new IAE("Function[%s] needs 2 arguments", name()); } final ExprEval eval = args.get(0).eval(bindings); - return eval.isNull() ? args.get(1).eval(bindings) : eval; + return eval.value() == null ? args.get(1).eval(bindings) : eval; } } @@ -1094,7 +1094,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final ExprEval expr = args.get(0).eval(bindings); - return ExprEval.of(expr.isNull(), ExprType.LONG); + return ExprEval.of(expr.value() == null, ExprType.LONG); } } @@ -1114,7 +1114,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final ExprEval expr = args.get(0).eval(bindings); - return ExprEval.of(!expr.isNull(), ExprType.LONG); + return ExprEval.of(expr.value() != null, ExprType.LONG); } } } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 800bc5740c28..01c854be572a 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; -import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -140,11 +139,7 @@ public void testLongEval() Assert.assertEquals(1271055781L, evalLong("unix_timestamp('2010-04-12T07:03:01')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - if (NullHandling.replaceWithDefault()) { - Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); - } else { - Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); - } + Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); Assert.assertEquals("x", eval("nvl(if(x == 9223372036854775806, '', 'x'), 'NULL')", bindings).asString()); } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index fe28ce163ffe..5ba40191218b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -167,7 +167,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public boolean isNull() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval.isNull(); + return exprEval == null || exprEval.isNull(); } } return new ExpressionFloatColumnSelector(); @@ -208,7 +208,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public boolean isNull() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval.isNull(); + return exprEval == null || exprEval.isNull(); } } return new ExpressionLongColumnSelector(); @@ -249,7 +249,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public boolean isNull() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval.isNull(); + return exprEval == null || exprEval.isNull(); } } return new ExpressionDoubleColumnSelector(); diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 7589065ce887..1a311e9fbc7b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -87,7 +87,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSele @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { - return new LongMaxBufferAggregator(selector); + return new LongMinBufferAggregator(selector); } private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index a077ccaf7443..1852037296ae 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -39,10 +39,6 @@ public final Aggregator factorize(ColumnSelectorFactory metricFactory) return NullHandling.replaceWithDefault() ? aggregator : new NullableAggregator(aggregator, selector); } - protected abstract ColumnValueSelector selector(ColumnSelectorFactory metricFactory); - - protected abstract Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); - @Override public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { @@ -51,7 +47,10 @@ public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFact return NullHandling.replaceWithDefault() ? aggregator : new NullableBufferAggregator(aggregator, selector); } - protected abstract BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); + protected abstract BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, + ColumnValueSelector selector + ); @Override public final AggregateCombiner makeAggregateCombiner() @@ -60,13 +59,20 @@ public final AggregateCombiner makeAggregateCombiner() return NullHandling.replaceWithDefault() ? combiner : new NullableAggregateCombiner(combiner); } - protected abstract AggregateCombiner makeAggregateCombiner2(); - @Override public final int getMaxIntermediateSize() { return getMaxIntermediateSize2() + (NullHandling.replaceWithDefault() ? 0 : Byte.BYTES); } + // ---- ABSTRACT METHODS BELOW ------ + + protected abstract ColumnValueSelector selector(ColumnSelectorFactory metricFactory); + + protected abstract Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); + + protected abstract AggregateCombiner makeAggregateCombiner2(); + protected abstract int getMaxIntermediateSize2(); + } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index 1c2605fe9853..57dc967e1af2 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.base.Throwables; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionCacheHelper; import io.druid.query.extraction.FunctionalExtraction; @@ -52,9 +52,9 @@ public LookupExtractionFn( { @Nullable @Override - public String apply(String input) + public String apply(@Nullable String input) { - return lookup.apply(Strings.nullToEmpty(input)); + return NullHandling.emptyToNullIfNeeded(lookup.apply(NullHandling.nullToEmptyIfNeeded(input))); } }, retainMissingValue, diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java index 9a96d28db49c..f3d4ef565fbc 100644 --- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.MapInputRowParser; @@ -48,6 +49,7 @@ import io.druid.segment.IndexBuilder; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.After; import org.junit.Assert; @@ -284,8 +286,13 @@ public void testNumericEvolutionTimeseriesAggregation() ); // Only nonexistent(4) + Map result = Maps.newHashMap(); + result.put("a", NullHandling.defaultLongValue()); + result.put("b", NullHandling.defaultDoubleValue()); + result.put("c", NullHandling.defaultLongValue()); + result.put("d", NullHandling.defaultDoubleValue()); Assert.assertEquals( - timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), + timeseriesResult(result), runQuery(query, factory, ImmutableList.of(index4)) ); @@ -354,7 +361,14 @@ public void testNumericEvolutionFiltering() // Only nonexistent(4) Assert.assertEquals( - timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L)), + timeseriesResult(TestHelper.createExpectedMap( + "a", + NullHandling.defaultLongValue(), + "b", + NullHandling.defaultDoubleValue(), + "c", + 0L + )), runQuery(query, factory, ImmutableList.of(index4)) ); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java index a33d99b41086..bf33e099ab2f 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java @@ -56,7 +56,7 @@ public void setup() @Test public void testDoubleMaxAggregator() { - DoubleMaxAggregator agg = (DoubleMaxAggregator) doubleMaxAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleMaxAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -71,9 +71,9 @@ public void testDoubleMaxAggregator() @Test public void testDoubleMaxBufferAggregator() { - DoubleMaxBufferAggregator agg = (DoubleMaxBufferAggregator) doubleMaxAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = doubleMaxAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Double.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Double.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -105,13 +105,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMaxAggregator agg) + private void aggregate(TestDoubleColumnSelectorImpl selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMaxBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestDoubleColumnSelectorImpl selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java index af72ee391d84..bad5331396e7 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java @@ -56,7 +56,7 @@ public void setup() @Test public void testDoubleMinAggregator() { - DoubleMinAggregator agg = (DoubleMinAggregator) doubleMinAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleMinAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -71,9 +71,9 @@ public void testDoubleMinAggregator() @Test public void testDoubleMinBufferAggregator() { - DoubleMinBufferAggregator agg = (DoubleMinBufferAggregator) doubleMinAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = doubleMinAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Double.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Double.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -105,13 +105,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMinAggregator agg) + private void aggregate(TestDoubleColumnSelectorImpl selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMinBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestDoubleColumnSelectorImpl selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 584f08a2b810..ed5ad443f5cb 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.js.JavaScriptConfig; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; @@ -224,9 +225,9 @@ public ColumnCapabilities getColumnCapabilities(String columnName) private void assertValues(FilteredAggregator agg, TestFloatColumnSelector selector, double... expectedVals) { - Assert.assertEquals(0.0d, agg.get()); - Assert.assertEquals(0.0d, agg.get()); - Assert.assertEquals(0.0d, agg.get()); + Assert.assertEquals(NullHandling.defaultDoubleValue(), agg.get()); + Assert.assertEquals(NullHandling.defaultDoubleValue(), agg.get()); + Assert.assertEquals(NullHandling.defaultDoubleValue(), agg.get()); for (double expectedVal : expectedVals) { aggregate(selector, agg); Assert.assertEquals(expectedVal, agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java index c790e8a3330d..1fa112f1a7d8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java @@ -56,7 +56,7 @@ public void setup() @Test public void testLongMaxAggregator() { - LongMaxAggregator agg = (LongMaxAggregator) longMaxAggFactory.factorize(colSelectorFactory); + Aggregator agg = longMaxAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -71,9 +71,9 @@ public void testLongMaxAggregator() @Test public void testLongMaxBufferAggregator() { - LongMaxBufferAggregator agg = (LongMaxBufferAggregator) longMaxAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = longMaxAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Long.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Long.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -105,13 +105,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestLongColumnSelector selector, LongMaxAggregator agg) + private void aggregate(TestLongColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestLongColumnSelector selector, LongMaxBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestLongColumnSelector selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java index 8298b2920b75..6e4747c65634 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java @@ -56,7 +56,7 @@ public void setup() @Test public void testLongMinAggregator() { - LongMinAggregator agg = (LongMinAggregator) longMinAggFactory.factorize(colSelectorFactory); + Aggregator agg = longMinAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -71,9 +71,9 @@ public void testLongMinAggregator() @Test public void testLongMinBufferAggregator() { - LongMinBufferAggregator agg = (LongMinBufferAggregator) longMinAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = longMinAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Long.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Long.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -105,13 +105,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestLongColumnSelector selector, LongMinAggregator agg) + private void aggregate(TestLongColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestLongColumnSelector selector, LongMinBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestLongColumnSelector selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index e7ddc3f94d1b..d0a6338b3ad8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.js.JavaScriptConfig; import io.druid.query.ColumnSelectorPlus; @@ -429,8 +430,8 @@ public void testAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { aggregate(selectorList, agg); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); - Assert.assertEquals(7L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); } @Test @@ -473,8 +474,8 @@ public void testBufferAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { bufferAggregate(selectorList, agg, buf, pos); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); - Assert.assertEquals(7L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); } @Test @@ -553,11 +554,11 @@ public void testCombineValues() aggregate(selector2, agg2); } - Assert.assertEquals(4.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 4.0 : 3.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); Assert.assertEquals( - 7.0, + NullHandling.replaceWithDefault() ? 7.0 : 6.0, (Double) rowAggregatorFactory.finalizeComputation( rowAggregatorFactory.combine( agg1.get(), diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 8aba7090aba7..15d419e06d19 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestDoubleColumnSelectorImpl; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleFirstAggregator() { - DoubleFirstAggregator agg = (DoubleFirstAggregator) doubleFirstAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { - DoubleFirstBufferAggregator agg = (DoubleFirstBufferAggregator) doubleFirstAggFactory.factorizeBuffered( + BufferAggregator agg = doubleFirstAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleFirstAggFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleFirstCombiningAggregator() { - DoubleFirstAggregator agg = (DoubleFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { - DoubleFirstBufferAggregator agg = (DoubleFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleFirstAggFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - DoubleFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - DoubleFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index ff0f00f5eb0b..719c16980dd6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestFloatColumnSelector; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleFirstAggregator() { - FloatFirstAggregator agg = (FloatFirstAggregator) floatFirstAggregatorFactory.factorize(colSelectorFactory); + Aggregator agg = floatFirstAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { - FloatFirstBufferAggregator agg = (FloatFirstBufferAggregator) floatFirstAggregatorFactory.factorizeBuffered( + BufferAggregator agg = floatFirstAggregatorFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleFirstCombiningAggregator() { - FloatFirstAggregator agg = (FloatFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { - FloatFirstBufferAggregator agg = (FloatFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - FloatFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - FloatFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index 36a8b5cb3f6a..6c48dbfd5ac1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -70,7 +72,7 @@ public void setup() @Test public void testLongFirstAggregator() { - LongFirstAggregator agg = (LongFirstAggregator) longFirstAggFactory.factorize(colSelectorFactory); + Aggregator agg = longFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -88,7 +90,7 @@ public void testLongFirstAggregator() @Test public void testLongFirstBufferAggregator() { - LongFirstBufferAggregator agg = (LongFirstBufferAggregator) longFirstAggFactory.factorizeBuffered( + BufferAggregator agg = longFirstAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longFirstAggFactory.getMaxIntermediateSize()]); @@ -118,7 +120,7 @@ public void testCombine() @Test public void testLongFirstCombiningAggregator() { - LongFirstAggregator agg = (LongFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -137,7 +139,7 @@ public void testLongFirstCombiningAggregator() @Test public void testLongFirstCombiningBufferAggregator() { - LongFirstBufferAggregator agg = (LongFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longFirstAggFactory.getMaxIntermediateSize()]); @@ -167,7 +169,7 @@ public void testSerde() throws Exception } private void aggregate( - LongFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -177,7 +179,7 @@ private void aggregate( } private void aggregate( - LongFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index c5453e18171a..423472df77b1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestDoubleColumnSelectorImpl; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleLastAggregator() { - DoubleLastAggregator agg = (DoubleLastAggregator) doubleLastAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleLastAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { - DoubleLastBufferAggregator agg = (DoubleLastBufferAggregator) doubleLastAggFactory.factorizeBuffered( + BufferAggregator agg = doubleLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleLastAggFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleLastCombiningAggregator() { - DoubleLastAggregator agg = (DoubleLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { - DoubleLastBufferAggregator agg = (DoubleLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleLastAggFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - DoubleLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - DoubleLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index 1938f8387339..2179f8feb809 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestFloatColumnSelector; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleLastAggregator() { - FloatLastAggregator agg = (FloatLastAggregator) floatLastAggregatorFactory.factorize(colSelectorFactory); + Aggregator agg = floatLastAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { - FloatLastBufferAggregator agg = (FloatLastBufferAggregator) floatLastAggregatorFactory.factorizeBuffered( + BufferAggregator agg = floatLastAggregatorFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleLastCombiningAggregator() { - FloatLastAggregator agg = (FloatLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { - FloatLastBufferAggregator agg = (FloatLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - FloatLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - FloatLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index 25d9160e15f7..5106749298c8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -70,7 +72,7 @@ public void setup() @Test public void testLongLastAggregator() { - LongLastAggregator agg = (LongLastAggregator) longLastAggFactory.factorize(colSelectorFactory); + Aggregator agg = longLastAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -88,7 +90,7 @@ public void testLongLastAggregator() @Test public void testLongLastBufferAggregator() { - LongLastBufferAggregator agg = (LongLastBufferAggregator) longLastAggFactory.factorizeBuffered( + BufferAggregator agg = longLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longLastAggFactory.getMaxIntermediateSize()]); @@ -118,7 +120,7 @@ public void testCombine() @Test public void testLongLastCombiningAggregator() { - LongLastAggregator agg = (LongLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -137,7 +139,7 @@ public void testLongLastCombiningAggregator() @Test public void testLongLastCombiningBufferAggregator() { - LongLastBufferAggregator agg = (LongLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longLastAggFactory.getMaxIntermediateSize()]); @@ -167,7 +169,7 @@ public void testSerde() throws Exception } private void aggregate( - LongLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -177,7 +179,7 @@ private void aggregate( } private void aggregate( - LongLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java index 609e74f375d4..42560206a52c 100644 --- a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java +++ b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -135,7 +136,7 @@ public void testRetainMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -149,7 +150,7 @@ public void testRetainMissingButFound() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -163,7 +164,11 @@ public void testReplaceMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? MISSING : out, exFn.apply(in)); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? MISSING : out, exFn.apply(in)); + } else { + Assert.assertEquals(out == null ? MISSING : out, exFn.apply(in)); + } } @@ -178,7 +183,11 @@ public void testReplaceMissingBlank() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + } else { + Assert.assertEquals(out == null ? "" : out, exFn.apply(in)); + } } @Test @@ -192,7 +201,11 @@ public void testOnlyOneValuePresent() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + } else { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? "" : out, exFn.apply(in)); + } } @Test @@ -204,7 +217,7 @@ public void testNullInputs() null, false ); - if (Strings.isNullOrEmpty(fn.apply(null))) { + if (NullHandling.isNullOrEquivalent(fn.apply(null))) { Assert.assertEquals(null, exFn.apply(null)); } } diff --git a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java index dc4494dadbde..ef4dfe84117d 100644 --- a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import io.druid.js.JavaScriptConfig; @@ -131,7 +132,11 @@ public void testJavascriptIsNull() Assert.assertEquals("yes", extractionFn.apply((String) null)); Assert.assertEquals("yes", extractionFn.apply((Object) null)); - Assert.assertEquals("yes", extractionFn.apply("")); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals("yes", extractionFn.apply("")); + } else { + Assert.assertEquals("no", extractionFn.apply("")); + } Assert.assertEquals("no", extractionFn.apply("abc")); Assert.assertEquals("no", extractionFn.apply(new Object())); Assert.assertEquals("no", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java index 1a7727adbaf1..fc014fa918a5 100644 --- a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java @@ -19,6 +19,7 @@ package io.druid.query.extraction; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,7 @@ public class LowerExtractionFnTest public void testApply() { Assert.assertEquals("lower 1 string", extractionFn.apply("lOwER 1 String")); - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java index 4415056f542a..14c0d293660b 100644 --- a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -75,7 +76,7 @@ public void testNullExtraction() Assert.assertNull(extractionFn.apply((Object) null)); Assert.assertNull(extractionFn.apply((String) null)); - Assert.assertNull(extractionFn.apply((String) "")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply((String) "")); } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java index ce6e468ccd93..a08f6686b6d0 100644 --- a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -145,11 +146,11 @@ public void testNullAndEmpty() String regex = "(.*)/.*/.*"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, false, null); // no match, map empty input value to null - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("")); // null value, returns null Assert.assertEquals(null, extractionFn.apply(null)); // empty match, map empty result to null - Assert.assertEquals(null, extractionFn.apply("/a/b")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("/a/b")); } @Test @@ -168,8 +169,8 @@ public void testMissingValueReplacementWhenPatternMatchesNull() { String regex = "^()$"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, "NO MATCH"); - Assert.assertEquals(null, extractionFn.apply("")); - Assert.assertEquals(null, extractionFn.apply(null)); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "NO MATCH", extractionFn.apply(null)); Assert.assertEquals("NO MATCH", extractionFn.apply("abc")); } @@ -178,10 +179,10 @@ public void testMissingValueReplacementToEmpty() { String regex = "(bob)"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, ""); - Assert.assertEquals(null, extractionFn.apply(null)); - Assert.assertEquals(null, extractionFn.apply("")); - Assert.assertEquals(null, extractionFn.apply("abc")); - Assert.assertEquals(null, extractionFn.apply("123")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply(null)); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("abc")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("123")); Assert.assertEquals("bob", extractionFn.apply("bobby")); } diff --git a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java index 08b102c6cfb2..d83388a75702 100644 --- a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java @@ -20,6 +20,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -29,7 +30,7 @@ public class StrlenExtractionFnTest @Test public void testApply() { - Assert.assertEquals("0", StrlenExtractionFn.instance().apply(null)); + Assert.assertEquals(NullHandling.replaceWithDefault() ? "0" : null, StrlenExtractionFn.instance().apply(null)); Assert.assertEquals("0", StrlenExtractionFn.instance().apply("")); Assert.assertEquals("1", StrlenExtractionFn.instance().apply("x")); Assert.assertEquals("3", StrlenExtractionFn.instance().apply("foo")); diff --git a/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java index 77fb05cdb2e4..18f91a0e146b 100644 --- a/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -46,7 +47,11 @@ public void testEmptyAndNullExtraction() ExtractionFn extractionFn = new TimeDimExtractionFn("MM/dd/yyyy", "MM/yyyy"); Assert.assertNull(extractionFn.apply(null)); - Assert.assertNull(extractionFn.apply("")); + if (NullHandling.replaceWithDefault()) { + Assert.assertNull(extractionFn.apply("")); + } else { + Assert.assertEquals("", extractionFn.apply("")); + } } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java index 1d038c0674ac..fd23548d2b61 100644 --- a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java @@ -19,6 +19,7 @@ package io.druid.query.extraction; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,7 @@ public class UpperExtractionFnTest public void testApply() { Assert.assertEquals("UPPER", extractionFn.apply("uPpeR")); - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 55e934642c7c..86038597ba06 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -35,6 +35,7 @@ import io.druid.collections.DefaultBlockingPool; import io.druid.collections.NonBlockingPool; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; @@ -7011,12 +7012,23 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() .setGranularity(QueryRunnerTestHelper.dayGran) .setDimFilter(new ExtractionDimFilter("quality", "", lookupExtractionFn, null)) .build(); - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) - ); + + List expectedResults; + + if (NullHandling.replaceWithDefault()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + ); + } else { + // Only empty string should match, nulls will not match + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); @@ -7065,10 +7077,17 @@ public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() public void testGroupByWithExtractionDimFilterKeyisNull() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULLorEMPTY"); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.replaceWithDefault()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + extractionMap.put("", "REPLACED_VALUE"); + } else { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "REPLACED_VALUE", true, false); + extractionMap.put("", "NOT_USED"); + } GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -7090,7 +7109,7 @@ public void testGroupByWithExtractionDimFilterKeyisNull() .setDimFilter( new ExtractionDimFilter( "null_column", - "NULLorEMPTY", + "REPLACED_VALUE", lookupExtractionFn, null ) @@ -7146,25 +7165,137 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() .setGranularity(QueryRunnerTestHelper.dayGran) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "automotive", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "business", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "health", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 0L, "idx", 0L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "premium", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "technology", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "travel", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "automotive", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "business", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "entertainment", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "health", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 0L, "idx", 0L) + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "premium", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "technology", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "travel", + "rows", + 0L, + "idx", + NullHandling.defaultLongValue() + ) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -7219,7 +7350,14 @@ public void testGroupByWithExtractionDimFilterNullDims() extractionMap.put("", "EMPTY"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.replaceWithDefault()) { + extractionMap.put("", "EMPTY"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + } else { + extractionMap.put("", "SHOULD_NOT_BE_USED"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "EMPTY", true, true); + } GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -8208,12 +8346,13 @@ public void testGroupByNumericStringsAsNumericWithDecoration() .addOrderByColumn("ql") .build(); + List expectedResults; // "entertainment" rows are excluded by the decorated specs, they become empty rows - List expectedResults = Arrays.asList( + expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", - "ql", 0L, - "qf", 0.0, + "ql", NullHandling.defaultLongValue(), + "qf", NullHandling.defaultDoubleValue(), "count", 2L ), GroupByQueryRunnerTestHelper.createExpectedRow( @@ -8266,21 +8405,38 @@ public void testGroupByDecorationOnNumerics() ) .setGranularity(QueryRunnerTestHelper.allGran) .build(); - - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 0L, - "qf", 0.0, - "count", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 1700L, - "qf", 17000.0, - "count", 2L - ) - ); + List expectedResults; + if (NullHandling.replaceWithDefault()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 0L, + "qf", 0.0, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 1700L, + "qf", 17000.0, + "count", 2L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", null, + "qf", null, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 1700L, + "qf", 17000.0, + "count", 2L + ) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java index d3bd541a66b2..37a8560dffb4 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Ordering; import com.google.common.io.Files; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.ByteBufferUtils; import io.druid.query.aggregation.AggregatorFactory; @@ -111,7 +112,7 @@ public void testGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, 2); - final int expectedMaxSize = 219; + final int expectedMaxSize = NullHandling.replaceWithDefault() ? 219 : 210; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -139,7 +140,7 @@ public void testGrowing2() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 2_000_000_000, 2); - final int expectedMaxSize = 40988516; + final int expectedMaxSize = NullHandling.replaceWithDefault() ? 40988516 : 39141224; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -153,7 +154,7 @@ public void testGrowing3() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, Integer.MAX_VALUE, 2); - final int expectedMaxSize = 44938972; + final int expectedMaxSize = NullHandling.replaceWithDefault() ? 44938972 : 42955456; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -167,7 +168,7 @@ public void testNoGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE); - final int expectedMaxSize = 267; + final int expectedMaxSize = NullHandling.replaceWithDefault() ? 267 : 258; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java index 6710b74bd8fc..815e862742a4 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.IAE; import io.druid.query.aggregation.AggregatorFactory; @@ -53,20 +54,37 @@ public void testLimitAndBufferSwapping() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i + keyBase), grouper.aggregate(i + keyBase).isOk()); } + if (NullHandling.replaceWithDefault()) { + // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes + // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes + // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes + // table arena is split in halves when doing push down, so each half is 9798 bytes + // each table arena half can hold 9798 / 28 = 349 buckets, with load factor of 0.5 max buckets per half is 174 + // First buffer swap occurs when we hit 174 buckets + // Subsequent buffer swaps occur after every 74 buckets, since we keep 100 buckets due to the limit + // With 1000 keys inserted, this results in one swap at the first 174 buckets, then 11 swaps afterwards. + // After the last swap, we have 100 keys + 12 new keys inserted. + Assert.assertEquals(12, grouper.getGrowthCount()); + Assert.assertEquals(112, grouper.getSize()); + Assert.assertEquals(349, grouper.getBuckets()); + Assert.assertEquals(174, grouper.getMaxSize()); + } else { + // With Nullability enabled + // bucket size is hash(int) + key(int) + aggs(2 longs + 1 bytes for Long Agg nullability) + heap offset(int) = 29 bytes + // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes + // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes + // table arena is split in halves when doing push down, so each half is 9798 bytes + // each table arena half can hold 9798 / 29 = 337 buckets, with load factor of 0.5 max buckets per half is 168 + // First buffer swap occurs when we hit 168 buckets + // Subsequent buffer swaps occur after every 68 buckets, since we keep 100 buckets due to the limit + // With 1000 keys inserted, this results in one swap at the first 169 buckets, then 12 swaps afterwards. + // After the last swap, we have 100 keys + 16 new keys inserted. + Assert.assertEquals(13, grouper.getGrowthCount()); + Assert.assertEquals(116, grouper.getSize()); + Assert.assertEquals(337, grouper.getBuckets()); + Assert.assertEquals(168, grouper.getMaxSize()); + } - // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes - // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes - // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes - // table arena is split in halves when doing push down, so each half is 9798 bytes - // each table arena half can hold 9798 / 28 = 349 buckets, with load factor of 0.5 max buckets per half is 174 - // First buffer swap occurs when we hit 174 buckets - // Subsequent buffer swaps occur after every 74 buckets, since we keep 100 buckets due to the limit - // With 1000 keys inserted, this results in one swap at the first 174 buckets, then 11 swaps afterwards. - // After the last swap, we have 100 keys + 12 new keys inserted. - Assert.assertEquals(12, grouper.getGrowthCount()); - Assert.assertEquals(112, grouper.getSize()); - Assert.assertEquals(349, grouper.getBuckets()); - Assert.assertEquals(174, grouper.getMaxSize()); Assert.assertEquals(100, grouper.getLimit()); // Aggregate slightly different row @@ -77,14 +95,27 @@ public void testLimitAndBufferSwapping() Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - // we added another 1000 unique keys - // previous size is 112, so next swap occurs after 62 rows - // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, - // with 50 keys being added after the final swap. - Assert.assertEquals(25, grouper.getGrowthCount()); - Assert.assertEquals(150, grouper.getSize()); - Assert.assertEquals(349, grouper.getBuckets()); - Assert.assertEquals(174, grouper.getMaxSize()); + if (NullHandling.replaceWithDefault()) { + // we added another 1000 unique keys + // previous size is 112, so next swap occurs after 62 rows + // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, + // with 50 keys being added after the final swap. + Assert.assertEquals(25, grouper.getGrowthCount()); + Assert.assertEquals(150, grouper.getSize()); + Assert.assertEquals(349, grouper.getBuckets()); + Assert.assertEquals(174, grouper.getMaxSize()); + } else { + // With Nullable Aggregator + // we added another 1000 unique keys + // previous size is 116, so next swap occurs after 52 rows + // after that, there are 1000 - 52 = 948 rows, 948 / 68 = 13 additional swaps after the first, + // with 64 keys being added after the final swap. + Assert.assertEquals(27, grouper.getGrowthCount()); + Assert.assertEquals(164, grouper.getSize()); + Assert.assertEquals(337, grouper.getBuckets()); + Assert.assertEquals(168, grouper.getMaxSize()); + } + Assert.assertEquals(100, grouper.getLimit()); final List> expected = Lists.newArrayList(); @@ -110,7 +141,7 @@ public void testMinBufferSize() final int limit = 100; final int keyBase = 100000; final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); - final LimitedBufferHashGrouper grouper = makeGrouper(columnSelectorFactory, 11716, 2, limit); + final LimitedBufferHashGrouper grouper = makeGrouper(columnSelectorFactory, 12120, 2, limit); final int numRows = 1000; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); @@ -119,10 +150,17 @@ public void testMinBufferSize() } // With minimum buffer size, after the first swap, every new key added will result in a swap - Assert.assertEquals(899, grouper.getGrowthCount()); - Assert.assertEquals(101, grouper.getSize()); - Assert.assertEquals(202, grouper.getBuckets()); - Assert.assertEquals(101, grouper.getMaxSize()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(224, grouper.getGrowthCount()); + Assert.assertEquals(104, grouper.getSize()); + Assert.assertEquals(209, grouper.getBuckets()); + Assert.assertEquals(104, grouper.getMaxSize()); + } else { + Assert.assertEquals(899, grouper.getGrowthCount()); + Assert.assertEquals(101, grouper.getSize()); + Assert.assertEquals(202, grouper.getBuckets()); + Assert.assertEquals(101, grouper.getMaxSize()); + } Assert.assertEquals(100, grouper.getLimit()); // Aggregate slightly different row @@ -132,11 +170,17 @@ public void testMinBufferSize() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - - Assert.assertEquals(1899, grouper.getGrowthCount()); - Assert.assertEquals(101, grouper.getSize()); - Assert.assertEquals(202, grouper.getBuckets()); - Assert.assertEquals(101, grouper.getMaxSize()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(474, grouper.getGrowthCount()); + Assert.assertEquals(104, grouper.getSize()); + Assert.assertEquals(209, grouper.getBuckets()); + Assert.assertEquals(104, grouper.getMaxSize()); + } else { + Assert.assertEquals(1899, grouper.getGrowthCount()); + Assert.assertEquals(101, grouper.getSize()); + Assert.assertEquals(202, grouper.getBuckets()); + Assert.assertEquals(101, grouper.getMaxSize()); + } Assert.assertEquals(100, grouper.getLimit()); final List> expected = Lists.newArrayList(); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java index e66cd77b0ac5..7458f53f34ad 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.concurrent.Execs; import io.druid.query.aggregation.AggregatorFactory; @@ -100,7 +101,7 @@ public void testStreamingAggregateWithLargeBuffer() throws ExecutionException, I @Test(timeout = 5000L) public void testStreamingAggregateWithMinimumBuffer() throws ExecutionException, InterruptedException { - testStreamingAggregate(60); + testStreamingAggregate(83); } private void testStreamingAggregate(int bufferSize) throws ExecutionException, InterruptedException @@ -128,7 +129,10 @@ private void testStreamingAggregate(int bufferSize) throws ExecutionException, I }); final List> unsortedEntries = Lists.newArrayList(grouper.iterator(true)); - final List> actual = Ordering.from((Comparator>) (o1, o2) -> Ints.compare(o1.getKey(), o2.getKey())) + final List> actual = Ordering.from((Comparator>) (o1, o2) -> Ints.compare( + o1.getKey(), + o2.getKey() + )) .sortedCopy(unsortedEntries); if (!actual.equals(expected)) { @@ -145,7 +149,11 @@ private void testStreamingAggregate(int bufferSize) throws ExecutionException, I public void testNotEnoughBuffer() { expectedException.expect(IllegalStateException.class); - expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[20]"); + if (NullHandling.replaceWithDefault()) { + expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[20]"); + } else { + expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[21]"); + } newGrouper(GrouperTestUtil.newColumnSelectorFactory(), 50); } @@ -157,7 +165,7 @@ public void testTimeout() expectedException.expectCause(CoreMatchers.instanceOf(TimeoutException.class)); final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); - final StreamingMergeSortedGrouper grouper = newGrouper(columnSelectorFactory, 60); + final StreamingMergeSortedGrouper grouper = newGrouper(columnSelectorFactory, 100); columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); grouper.aggregate(6); diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java index a363f7606039..004b33258b06 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java @@ -20,6 +20,7 @@ package io.druid.query.lookup; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.query.extraction.MapLookupExtractor; import org.junit.Assert; import org.junit.Test; @@ -65,7 +66,11 @@ public void testNullKeyIsMappable() false, false ); - Assert.assertEquals("bar", lookupExtractionFn.apply(null)); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals("bar", lookupExtractionFn.apply(null)); + } else { + Assert.assertEquals("REPLACE", lookupExtractionFn.apply(null)); + } } @Test diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index d53c55396dfd..0c9d850dfcd4 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -752,7 +753,7 @@ public void testSearchWithNullValueInDimension() throws Exception QueryRunner runner = factory.createRunner(new QueryableIndexSegment("asdf", TestIndex.persistRealtimeAndLoadMMapped(index))); List expectedHits = Lists.newLinkedList(); expectedHits.add(new SearchHit("table", "table", 1)); - expectedHits.add(new SearchHit("table", "", 1)); + expectedHits.add(new SearchHit("table", NullHandling.defaultStringValue(), 1)); checkSearchQuery(searchQuery, runner, expectedHits); } diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index bed640886452..babce9ccdd13 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -23,7 +23,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.primitives.Doubles; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; @@ -144,16 +146,15 @@ public void testEmptyTimeseries() ) .descending(descending) .build(); - + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.defaultDoubleValue()); + resultMap.put("first", NullHandling.defaultDoubleValue()); List> expectedResults = ImmutableList.of( new Result<>( DateTimes.of("2020-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0D, - "first", 0D - ) + resultMap ) ) ); @@ -205,24 +206,61 @@ public void testFullOnTimeseries() QueryRunnerTestHelper.skippedDay.equals(current) ? 0L : 13L, value.getLongMetric("rows").longValue() ); - Assert.assertEquals( - result.toString(), - Doubles.tryParse(expectedIndex[count]).doubleValue(), - value.getDoubleMetric("index").doubleValue(), - value.getDoubleMetric("index").doubleValue() * 1e-6 - ); - Assert.assertEquals( - result.toString(), - new Double(expectedIndex[count]) + - (QueryRunnerTestHelper.skippedDay.equals(current) ? 0L : 13L) + 1L, - value.getDoubleMetric("addRowsIndexConstant"), - value.getDoubleMetric("addRowsIndexConstant") * 1e-6 - ); - Assert.assertEquals( - value.getDoubleMetric("uniques"), - QueryRunnerTestHelper.skippedDay.equals(current) ? 0.0d : 9.0d, - 0.02 - ); + + if (!QueryRunnerTestHelper.skippedDay.equals(current)) { + Assert.assertEquals( + result.toString(), + Doubles.tryParse(expectedIndex[count]).doubleValue(), + value.getDoubleMetric("index").doubleValue(), + value.getDoubleMetric("index").doubleValue() * 1e-6 + ); + Assert.assertEquals( + result.toString(), + new Double(expectedIndex[count]) + + 13L + 1L, + value.getDoubleMetric("addRowsIndexConstant"), + value.getDoubleMetric("addRowsIndexConstant") * 1e-6 + ); + Assert.assertEquals( + value.getDoubleMetric("uniques"), + 9.0d, + 0.02 + ); + } else { + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + result.toString(), + 0.0D, + value.getDoubleMetric("index").doubleValue(), + value.getDoubleMetric("index").doubleValue() * 1e-6 + ); + Assert.assertEquals( + result.toString(), + new Double(expectedIndex[count]) + 1L, + value.getDoubleMetric("addRowsIndexConstant"), + value.getDoubleMetric("addRowsIndexConstant") * 1e-6 + ); + Assert.assertEquals( + 0.0D, + value.getDoubleMetric("uniques"), + 0.02 + ); + } else { + Assert.assertNull( + result.toString(), + value.getDoubleMetric("index") + ); + Assert.assertNull( + result.toString(), + value.getDoubleMetric("addRowsIndexConstant") + ); + Assert.assertEquals( + value.getDoubleMetric("uniques"), + 0.0d, + 0.02 + ); + } + } lastResult = result; ++count; @@ -626,14 +664,15 @@ public void testTimeseriesQueryZeroFilling() final Iterable iterable = Granularities.HOUR.getIterable( new Interval(DateTimes.of("2011-04-14T01"), DateTimes.of("2011-04-15")) ); + Map noRowsResult = Maps.newHashMap(); + noRowsResult.put("rows", 0L); + noRowsResult.put("idx", NullHandling.defaultLongValue()); for (Interval interval : iterable) { lotsOfZeroes.add( - new Result<>( - interval.getStart(), - new TimeseriesResultValue( - ImmutableMap.of("rows", 0L, "idx", 0L) - ) - ) + new Result<>( + interval.getStart(), + new TimeseriesResultValue(noRowsResult) + ) ); } @@ -1338,27 +1377,23 @@ public void testTimeseriesWithFilterOnNonExistentDimension() .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.defaultDoubleValue()); + resultMap.put("addRowsIndexConstant", NullHandling.replaceWithDefault() ? 1.0 : null); + resultMap.put("uniques", 0.0); + List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -1483,28 +1518,23 @@ public void testTimeseriesWithNonExistentFilter() .postAggregators(QueryRunnerTestHelper.addRowsIndexConstant) .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.defaultDoubleValue()); + resultMap.put("addRowsIndexConstant", NullHandling.replaceWithDefault() ? 1.0 : null); + resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -1529,28 +1559,23 @@ public void testTimeseriesWithNonExistentFilterAndMultiDim() .postAggregators(QueryRunnerTestHelper.addRowsIndexConstant) .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.defaultDoubleValue()); + resultMap.put("addRowsIndexConstant", NullHandling.replaceWithDefault() ? 1.0 : null); + resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 1185c8044d3f..a198f68f867c 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -29,6 +29,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; @@ -4206,10 +4207,15 @@ public void testTopNWithExtractionFilter() public void testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULL"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.replaceWithDefault()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + extractionMap.put("", "NULL"); + } else { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, false); + } DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) @@ -4279,10 +4285,16 @@ private Sequence> runWithPreMergeAndMerge(TopNQuery quer public void testTopNWithExtractionFilterNoExistingValue() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULL"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.replaceWithDefault()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + extractionMap.put("", "NULL"); + } else { + extractionMap.put("", "NOT_USED"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, true); + } DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) diff --git a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java index abac95cf0159..fec961aec2f4 100644 --- a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.common.config.NullHandling; import io.druid.query.extraction.StringFormatExtractionFn; import io.druid.query.extraction.SubstringDimExtractionFn; import io.druid.segment.data.IndexedInts; @@ -68,7 +69,7 @@ public void testLookupName() throws Exception public void testLookupId() throws Exception { Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId(null)); - Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId("")); + Assert.assertEquals(NullHandling.replaceWithDefault() ? 0 : -1, NULL_SELECTOR.idLookup().lookupId("")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("billy")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("bob")); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java index 83bf29cf20ab..85337189dbbd 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.Pair; @@ -400,13 +401,13 @@ public void testNonIntersectingSchemas() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 2L) - .put("index", 100.0D) - .put("addRowsIndexConstant", 103.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 2L) + .put("index", 100.0D) + .put("addRowsIndexConstant", 103.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.replaceWithDefault() ? 0.0D : 100.0D) + .build() ) ) ); @@ -747,13 +748,13 @@ public void testValueAndEmptySchemas() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 2L) - .put("index", 100.0D) - .put("addRowsIndexConstant", 103.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 2L) + .put("index", 100.0D) + .put("addRowsIndexConstant", 103.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.replaceWithDefault() ? 0.0D : 100.0D) + .build() ) ) ); @@ -869,15 +870,14 @@ public void testEmptySchemas() new Result<>( DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 1L) - .put("index", 0.0D) - .put("addRowsIndexConstant", 2.0D) - .put("uniques", 0.0D) - .put("maxIndex", 0.0D) - .put("minIndex", 0.0D) - .build() - ) + TestHelper.createExpectedMap( + "rows", 1L, + "index", NullHandling.replaceWithDefault() ? 0.0D : null, + "addRowsIndexConstant", NullHandling.replaceWithDefault() ? 2.0D : null, + "uniques", 0.0D, + "maxIndex", NullHandling.replaceWithDefault() ? 0.0D : null, + "minIndex", NullHandling.replaceWithDefault() ? 0.0D : null + )) ) ); @@ -885,14 +885,14 @@ public void testEmptySchemas() new Result<>( DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 0L) - .put("index", 0.0D) - .put("addRowsIndexConstant", 1.0D) - .put("uniques", 0.0D) - .put("maxIndex", Double.NEGATIVE_INFINITY) - .put("minIndex", Double.POSITIVE_INFINITY) - .build() + TestHelper.createExpectedMap( + "rows", 0L, + "index", NullHandling.replaceWithDefault() ? 0.0D : null, + "addRowsIndexConstant", NullHandling.replaceWithDefault() ? 1.0D : null, + "uniques", 0.0D, + "maxIndex", NullHandling.replaceWithDefault() ? Double.NEGATIVE_INFINITY : null, + "minIndex", NullHandling.replaceWithDefault() ? Double.POSITIVE_INFINITY : null + ) ) ) ); @@ -1186,13 +1186,13 @@ public void testDifferentMetrics() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 11L) - .put("index", 900.0D) - .put("addRowsIndexConstant", 912.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 11L) + .put("index", 900.0D) + .put("addRowsIndexConstant", 912.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.replaceWithDefault() ? 0.0D : 100.0D) + .build() ) ) ); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index 2c04eca92b7d..6855d29d716d 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; @@ -151,7 +152,7 @@ public void testFullOnTimeseries() .put("addRowsIndexConstant", 912.0) .put("uniques", 2.000977198748901D) .put("maxIndex", 100.0) - .put("minIndex", 0.0) + .put("minIndex", NullHandling.replaceWithDefault() ? 0.0 : 100.0) .build() ) ) diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index 976aa9fecfd4..4fbd2b7721c9 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -21,7 +21,9 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.jackson.DefaultObjectMapper; @@ -315,12 +317,16 @@ private static void assertRow(String msg, Row expected, Row actual) final Object actualValue = actualMap.get(key); if (expectedValue instanceof Float || expectedValue instanceof Double) { - Assert.assertEquals( - StringUtils.format("%s: key[%s]", msg, key), - ((Number) expectedValue).doubleValue(), - ((Number) actualValue).doubleValue(), - Math.abs(((Number) expectedValue).doubleValue() * 1e-6) - ); + if (expectedValue == null) { + Assert.assertNull(actualValue); + } else { + Assert.assertEquals( + StringUtils.format("%s: key[%s]", msg, key), + ((Number) expectedValue).doubleValue(), + ((Number) actualValue).doubleValue(), + Math.abs(((Number) expectedValue).doubleValue() * 1e-6) + ); + } } else { Assert.assertEquals( StringUtils.format("%s: key[%s]", msg, key), @@ -330,4 +336,16 @@ private static void assertRow(String msg, Row expected, Row actual) } } } + + + public static Map createExpectedMap(Object... vals) + { + Preconditions.checkArgument(vals.length % 2 == 0); + + Map theVals = Maps.newHashMap(); + for (int i = 0; i < vals.length; i += 2) { + theVals.put(vals[i].toString(), vals[i + 1]); + } + return theVals; + } } diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java index 0e6c7bb318ef..da8de3682c83 100644 --- a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -88,6 +89,21 @@ public static void tearDown() throws Exception @Test public void testLexicographicMatchEverything() + { + final List filters = ImmutableList.of( + new BoundDimFilter("dim0", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim1", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim2", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim3", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) + ); + + for (BoundDimFilter filter : filters) { + assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + } + } + + @Test + public void testLexicographicMatchWithEmptyString() { final List filters = ImmutableList.of( new BoundDimFilter("dim0", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), @@ -95,9 +111,15 @@ public void testLexicographicMatchEverything() new BoundDimFilter("dim2", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), new BoundDimFilter("dim3", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) ); - - for (BoundDimFilter filter : filters) { - assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + if (NullHandling.replaceWithDefault()) { + for (BoundDimFilter filter : filters) { + assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + } + } else { + assertFilterMatches(filters.get(0), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + assertFilterMatches(filters.get(1), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + assertFilterMatches(filters.get(2), ImmutableList.of("0", "2", "3", "4", "6", "7")); + assertFilterMatches(filters.get(3), ImmutableList.of()); } } @@ -112,19 +134,49 @@ public void testLexicographicMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("1", "2", "5") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("1", "2", "5") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("2") + ); + } } @Test public void testLexicographicMatchMissingColumn() { - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + assertFilterMatches( + new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } assertFilterMatches( new BoundDimFilter("dim3", "", "", true, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of() @@ -133,18 +185,11 @@ public void testLexicographicMatchMissingColumn() new BoundDimFilter("dim3", "", "", false, true, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of() ); - assertFilterMatches( - new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + assertFilterMatches( new BoundDimFilter("dim3", null, "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of() - ); } @@ -229,14 +274,25 @@ public void testAlphaNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of() + ); + } } @Test @@ -327,14 +383,26 @@ public void testNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.NUMERIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of() + ); + } + } @Test @@ -432,10 +500,17 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { return null; }"; ExtractionFn makeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches( - new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + } assertFilterMatches( new BoundDimFilter("dim1", "super-ab", "super-abd", true, true, false, superFn, StringComparators.LEXICOGRAPHIC), @@ -452,10 +527,69 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim2", "super-null", "super-null", false, false, false, superFn, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("1", "2", "5") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.NUMERIC + ), + ImmutableList.of("1", "2", "5") + ); + } else { + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim2", "super-", "super-", false, false, false, superFn, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim2", "super-", "super-", false, false, false, superFn, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + } assertFilterMatches( new BoundDimFilter("dim3", "super-null", "super-null", false, false, false, superFn, StringComparators.LEXICOGRAPHIC), @@ -467,11 +601,6 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim2", "super-null", "super-null", false, false, false, superFn, StringComparators.NUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( new BoundDimFilter("dim4", "super-null", "super-null", false, false, false, superFn, StringComparators.NUMERIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") diff --git a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java index 1b320c6ef8f2..c0d7043fe0fa 100644 --- a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -123,14 +124,28 @@ public void testMissingColumnNotSpecifiedInDimensionList() DefaultDimensionSpec.of("dim6"), DefaultDimensionSpec.of("dim7") )), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( - DefaultDimensionSpec.of("dim1"), - DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("0")); - assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( - DefaultDimensionSpec.of("dim2"), - DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("1", "2", "6", "7", "8")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim1"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("0")); + + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim2"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1", "2", "6", "7", "8")); + } else { + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim1"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of()); + + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim2"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1", "6", "7", "8")); + } + } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index 08cc28dd6a79..235e65147b91 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.FloatDimensionSchema; @@ -130,8 +131,13 @@ public void testOneMultiValuedStringColumn() { // Expressions currently treat multi-valued arrays as nulls. // This test is just documenting the current behavior, not necessarily saying it makes sense. - - assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); + } else { + assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("2")); + // AS per SQL standard null == null returns false. + assertFilterMatches(EDF("dim4 == null"), ImmutableList.of()); + } assertFilterMatches(EDF("dim4 == '1'"), ImmutableList.of()); assertFilterMatches(EDF("dim4 == '3'"), ImmutableList.of("3")); } @@ -188,12 +194,25 @@ public void testCompareColumns() @Test public void testMissingColumn() { - assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } else { + // AS per SQL standard null == null returns false. + assertFilterMatches(EDF("missing == null"), ImmutableList.of()); + } assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); - assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.replaceWithDefault()) { + // missing equivaluent to 0 + assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } else { + // missing equivalent to null + assertFilterMatches(EDF("missing < '2'"), ImmutableList.of()); + assertFilterMatches(EDF("missing < 2"), ImmutableList.of()); + assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of()); + } assertFilterMatches(EDF("missing > '2'"), ImmutableList.of()); assertFilterMatches(EDF("missing > 2"), ImmutableList.of()); assertFilterMatches(EDF("missing > 2.0"), ImmutableList.of()); diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index 5bd793a571b8..e45bc5d0397c 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -24,6 +24,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -116,7 +117,9 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { + //CHECKSTYLE.OFF: Regexp final String valueOrNull = Strings.emptyToNull(value); + //CHECKSTYLE.ON: Regexp final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override @@ -199,7 +202,11 @@ public static void tearDown() throws Exception @Test public void testSinglePreFilterWithNulls() { - assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + } else { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new SelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); @@ -212,7 +219,11 @@ public void testSinglePreFilterWithNulls() @Test public void testSinglePostFilterWithNulls() { - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of()); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "", null), ImmutableList.of("0")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); @@ -221,8 +232,11 @@ public void testSinglePostFilterWithNulls() assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "abc", null), ImmutableList.of("5", "8")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "ab", null), ImmutableList.of()); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN), ImmutableList.of("0")); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-10", JS_EXTRACTION_FN), ImmutableList.of("1")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), ImmutableList.of("2")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-1", JS_EXTRACTION_FN), ImmutableList.of("3", "9")); @@ -234,13 +248,23 @@ public void testSinglePostFilterWithNulls() @Test public void testBasicPreAndPostFilterWithNulls() { - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim2", "a", null), - new NoBitmapSelectorDimFilter("dim1", null, null) - )), - ImmutableList.of("0") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "a", null), + new NoBitmapSelectorDimFilter("dim1", null, null) + )), + ImmutableList.of("0") + ); + } else { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "a", null), + new NoBitmapSelectorDimFilter("dim1", null, null) + )), + ImmutableList.of() + ); + } assertFilterMatches( new AndDimFilter(Arrays.asList( @@ -274,14 +298,51 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of() ); - - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("0") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("0") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("2") + ); + } else { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("0") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of() + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("2") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of() + ); + } assertFilterMatches( new AndDimFilter(Arrays.asList( @@ -291,14 +352,6 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of("1") ); - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("2") - ); - assertFilterMatches( new AndDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-1", JS_EXTRACTION_FN), @@ -327,13 +380,23 @@ public void testOrPostFilterWithNulls() ImmutableList.of("0", "3") ); - assertFilterMatches( - new OrDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "abc", null), - new NoBitmapSelectorDimFilter("dim2", null, null) - )), - ImmutableList.of("1", "2", "5", "8") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "abc", null), + new NoBitmapSelectorDimFilter("dim2", null, null) + )), + ImmutableList.of("1", "2", "5", "8") + ); + } else { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "abc", null), + new NoBitmapSelectorDimFilter("dim2", null, null) + )), + ImmutableList.of("1", "5", "8") + ); + } assertFilterMatches( new OrDimFilter(Arrays.asList( @@ -382,13 +445,32 @@ public void testOrPostFilterWithNulls() )), ImmutableList.of("0", "3") ); - assertFilterMatches( - new OrDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("1", "2", "5", "8") - ); + + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("1", "2", "5", "8") + ); + } else { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("1", "5", "8") + ); + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("2", "5", "8") + ); + } + assertFilterMatches( new OrDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), @@ -432,7 +514,14 @@ public void testOrPostFilterWithNulls() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim3", "", null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + ); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of()); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "a", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "b", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "c", null), ImmutableList.of()); @@ -482,7 +571,17 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim4", "", null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + ); + } else { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim4", "", null), + ImmutableList.of() + ); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "a", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "b", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "c", null), ImmutableList.of()); diff --git a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java index d7e1a14293e0..58c98db48022 100644 --- a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -125,10 +126,17 @@ public void testSingleValueStringColumnWithNulls() ImmutableList.of("a") ); - assertFilterMatches( - toInFilter("dim1", null, "10", "abc"), - ImmutableList.of("a", "b", "f") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + toInFilter("dim1", null, "10", "abc"), + ImmutableList.of("a", "b", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim1", null, "10", "abc"), + ImmutableList.of("b", "f") + ); + } assertFilterMatches( toInFilter("dim1", "-1", "ab", "de"), @@ -139,28 +147,47 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches( - toInFilter("dim2", null), - ImmutableList.of("b", "c", "f") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + toInFilter("dim2", null), + ImmutableList.of("b", "c", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "a"), + ImmutableList.of("a", "b", "c", "d", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "b"), + ImmutableList.of("a", "b", "c", "f") + ); + assertFilterMatches( + toInFilter("dim2", ""), + ImmutableList.of("b", "c", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim2", null), + ImmutableList.of("b", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "a"), + ImmutableList.of("a", "b", "d", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "b"), + ImmutableList.of("a", "b", "f") + ); + assertFilterMatches( + toInFilter("dim2", ""), + ImmutableList.of("c") + ); + } assertFilterMatches( toInFilter("dim2", "", (String) null), ImmutableList.of("b", "c", "f") ); - assertFilterMatches( - toInFilter("dim2", null, "a"), - ImmutableList.of("a", "b", "c", "d", "f") - - ); - - assertFilterMatches( - toInFilter("dim2", null, "b"), - ImmutableList.of("a", "b", "c", "f") - - ); - assertFilterMatches( toInFilter("dim2", "c"), ImmutableList.of("e") @@ -180,10 +207,17 @@ public void testMissingColumn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - assertFilterMatches( - toInFilter("dim3", ""), - ImmutableList.of("a", "b", "c", "d", "e", "f") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + toInFilter("dim3", ""), + ImmutableList.of("a", "b", "c", "d", "e", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim3", ""), + ImmutableList.of() + ); + } assertFilterMatches( toInFilter("dim3", null, "a"), @@ -215,20 +249,43 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'YES'; } else { return 'NO';} }"; ExtractionFn yesNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches( - toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), - ImmutableList.of("a", "b", "c", "d", "f") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), + ImmutableList.of("a", "b", "c", "d", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), + ImmutableList.of("a", "b", "e") + ); + assertFilterMatches( + toInFilterWithFn("dim2", yesNullFn, "YES"), + ImmutableList.of("b", "c", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", yesNullFn, "NO"), + ImmutableList.of("b", "c", "d", "e", "f") + ); + } else { + assertFilterMatches( + toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), + ImmutableList.of("a", "b", "d", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), + ImmutableList.of("b", "e") + ); + assertFilterMatches( + toInFilterWithFn("dim2", yesNullFn, "YES"), + ImmutableList.of("b", "f") + ); + + assertFilterMatches( + toInFilterWithFn("dim1", yesNullFn, "NO"), + ImmutableList.of("a", "b", "c", "d", "e", "f") + ); + } - assertFilterMatches( - toInFilterWithFn("dim2", yesNullFn, "YES"), - ImmutableList.of("b", "c", "f") - ); - - assertFilterMatches( - toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), - ImmutableList.of("a", "b", "e") - ); assertFilterMatches( toInFilterWithFn("dim3", yesNullFn, "NO"), @@ -240,10 +297,6 @@ public void testMatchWithExtractionFn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - assertFilterMatches( - toInFilterWithFn("dim1", yesNullFn, "NO"), - ImmutableList.of("b", "c", "d", "e", "f") - ); } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java index 760ae0f7ad5a..3127f8dee025 100644 --- a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -109,7 +110,12 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); + } else { + assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of()); + assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter(""), null), ImmutableList.of("0")); + } assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("10"), null), ImmutableList.of("1")); assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("2"), null), ImmutableList.of("2")); assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("1"), null), ImmutableList.of("3")); @@ -122,7 +128,12 @@ public void testSingleValueStringColumnWithNulls() public void testMultiValueStringColumn() { // multi-val null...... - assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "5")); + assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter(""), null), ImmutableList.of("2")); + } assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("a"), null), ImmutableList.of("0", "3")); assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("b"), null), ImmutableList.of("0")); assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("c"), null), ImmutableList.of("4")); diff --git a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java index b9c127a81bc0..d97acb6105f9 100644 --- a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -156,10 +157,17 @@ public void testMatchEmptyString() @Test public void testMatchEmptyStringWithExtractionFn() { - assertFilterMatches( - new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), + ImmutableList.of() + ); + } } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java index 732cbbcb8f78..7dcc4c706474 100644 --- a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -99,7 +100,11 @@ public void testSingleValueStringColumnWithoutNulls() public void testSingleValueStringColumnWithNulls() { // RegexFilter always returns false for null row values. - assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } assertFilterMatches(new RegexDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new RegexDimFilter("dim1", "2", null), ImmutableList.of("2")); assertFilterMatches(new RegexDimFilter("dim1", "1", null), ImmutableList.of("1", "3")); @@ -111,7 +116,11 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); + } else { + assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "2", "3", "4")); + } assertFilterMatches(new RegexDimFilter("dim2", "a", null), ImmutableList.of("0", "3")); assertFilterMatches(new RegexDimFilter("dim2", "b", null), ImmutableList.of("0")); assertFilterMatches(new RegexDimFilter("dim2", "c", null), ImmutableList.of("4")); @@ -141,11 +150,16 @@ public void testRegexWithExtractionFn() { String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - - assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); + assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of()); + assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "5")); + } assertFilterMatches(new RegexDimFilter("dim1", "ab.*", changeNullFn), ImmutableList.of("4", "5")); - assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); + assertFilterMatches(new RegexDimFilter("dim2", "a.*", changeNullFn), ImmutableList.of("0", "3")); assertFilterMatches(new RegexDimFilter("dim3", ".*ANYMORE", changeNullFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java index b1cf8293457f..cc714c0d5fcc 100644 --- a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -105,8 +106,18 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - // SearchQueryFilter always returns false for null row values. - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue(""), null), ImmutableList.of("1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + // SearchQueryFilter always returns false for null row values. + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue(""), null), + ImmutableList.of("1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue(""), null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("10"), null), ImmutableList.of("1")); assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("2"), null), ImmutableList.of("2")); assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("1"), null), ImmutableList.of("1", "3")); @@ -118,7 +129,14 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue(""), null), + ImmutableList.of("0", "2", "3", "4") + ); + } assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("a"), null), ImmutableList.of("0", "3")); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("b"), null), ImmutableList.of("0")); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("c"), null), ImmutableList.of("4")); @@ -151,10 +169,31 @@ public void testSearchQueryWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0")); - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("ab"), changeNullFn), ImmutableList.of("4", "5")); - - assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("1", "2", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("0") + ); + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("1", "2", "5") + ); + + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of() + ); + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("1", "5") + ); + } + + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ab"), changeNullFn), + ImmutableList.of("4", "5") + ); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("a"), changeNullFn), ImmutableList.of("0", "3")); assertFilterMatches(new SearchQueryDimFilter("dim3", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index 4feac1adff95..68c3acc459dd 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -124,8 +125,13 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); - assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + } else { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); + assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + } assertFilterMatches(new SelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new SelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); assertFilterMatches(new SelectorDimFilter("dim1", "1", null), ImmutableList.of("3")); @@ -137,8 +143,13 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); - assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); + assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "5")); + assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("2")); + } assertFilterMatches(new SelectorDimFilter("dim2", "a", null), ImmutableList.of("0", "3")); assertFilterMatches(new SelectorDimFilter("dim2", "b", null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim2", "c", null), ImmutableList.of("4")); @@ -149,7 +160,11 @@ public void testMultiValueStringColumn() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim3", "a", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim3", "b", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim3", "c", null), ImmutableList.of()); @@ -159,7 +174,11 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim4", "a", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim4", "b", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim4", "c", null), ImmutableList.of()); @@ -211,7 +230,23 @@ public void testSelectorWithLookupExtractionFn() ); LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false); LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true); - assertFilterMatches(new SelectorDimFilter("dim0", null, lookupFn3), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + // Nulls and empty strings are considered equivalent + assertFilterMatches( + new SelectorDimFilter("dim0", null, lookupFn3), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new SelectorDimFilter("dim0", null, lookupFn3), + ImmutableList.of("0", "2", "3", "4", "5") + ); + assertFilterMatches( + new SelectorDimFilter("dim0", "", lookupFn3), + ImmutableList.of("1") + ); + } + final Map stringMap4 = ImmutableMap.of( "9", "4" @@ -252,7 +287,12 @@ public void testSelectorWithLookupExtractionFn() assertFilterMatches(optFilter1, ImmutableList.of("0", "1", "2", "5")); assertFilterMatches(optFilter2, ImmutableList.of("2", "5")); - assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + // Null and Empty strings are same + assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(optFilter3, ImmutableList.of("0", "2", "3", "4", "5")); + } assertFilterMatches(optFilter4, ImmutableList.of("5")); assertFilterMatches(optFilter5, ImmutableList.of()); assertFilterMatches(optFilter6, ImmutableList.of("5")); @@ -261,6 +301,20 @@ public void testSelectorWithLookupExtractionFn() // remove these when ExtractionDimFilter is removed. assertFilterMatches(new ExtractionDimFilter("dim1", "UNKNOWN", lookupFn, null), ImmutableList.of("0", "1", "2", "5")); assertFilterMatches(new ExtractionDimFilter("dim0", "5", lookupFn2, null), ImmutableList.of("2", "5")); - assertFilterMatches(new ExtractionDimFilter("dim0", null, lookupFn3, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches( + new ExtractionDimFilter("dim0", null, lookupFn3, null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new ExtractionDimFilter("dim0", null, lookupFn3, null), + ImmutableList.of("0", "2", "3", "4", "5") + ); + assertFilterMatches( + new ExtractionDimFilter("dim0", "", lookupFn3, null), + ImmutableList.of("1") + ); + } } } diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index 55ce4d17be33..19ba6c565eba 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -134,7 +134,11 @@ public void testLongSelector() final BaseLongColumnValueSelector selector = XPLUSY.makeColumnValueSelector("expr", COLUMN_SELECTOR_FACTORY); CURRENT_ROW.set(ROW0); - Assert.assertEquals(0L, selector.getLong()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0L, selector.getLong()); + } else { + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW1); if (NullHandling.replaceWithDefault()) { @@ -157,7 +161,11 @@ public void testLongSelectorUsingStringFunction() final BaseLongColumnValueSelector selector = ZCONCATX.makeColumnValueSelector("expr", COLUMN_SELECTOR_FACTORY); CURRENT_ROW.set(ROW0); - Assert.assertEquals(0L, selector.getLong()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0L, selector.getLong()); + } else { + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW1); if (NullHandling.replaceWithDefault()) { @@ -168,7 +176,11 @@ public void testLongSelectorUsingStringFunction() } CURRENT_ROW.set(ROW2); - Assert.assertEquals(0L, selector.getLong()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0L, selector.getLong()); + } else { + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW3); Assert.assertEquals(0L, selector.getLong()); @@ -180,7 +192,11 @@ public void testFloatSelector() final BaseFloatColumnValueSelector selector = XPLUSY.makeColumnValueSelector("expr", COLUMN_SELECTOR_FACTORY); CURRENT_ROW.set(ROW0); - Assert.assertEquals(0.0f, selector.getFloat(), 0.0f); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0.0f, selector.getFloat(), 0.0f); + } else { + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW1); if (NullHandling.replaceWithDefault()) { From 92b8451c152ace3c3bd4dda50d5d319cf782d73c Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 1 Mar 2018 01:26:37 +0530 Subject: [PATCH 04/38] druid-sql Null Handling --- .../calcite/expression/DruidExpression.java | 2 +- .../sql/calcite/expression/Expressions.java | 14 +- .../UnaryFunctionOperatorConversion.java | 66 +++ .../builtin/CeilOperatorConversion.java | 4 +- .../druid/sql/calcite/planner/Calcites.java | 2 + .../calcite/planner/DruidOperatorTable.java | 5 +- .../druid/sql/calcite/rel/DruidSemiJoin.java | 27 +- .../io/druid/sql/calcite/rel/QueryMaker.java | 4 +- .../druid/sql/avatica/DruidStatementTest.java | 9 +- .../druid/sql/calcite/CalciteQueryTest.java | 486 ++++++++++++++---- .../calcite/expression/ExpressionsTest.java | 27 +- .../sql/calcite/http/SqlResourceTest.java | 10 + .../sql/calcite/planner/CalcitesTest.java | 1 - 13 files changed, 503 insertions(+), 154 deletions(-) create mode 100644 sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index b779fdd03a0c..69c3915a3b41 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -90,7 +90,7 @@ public static String stringLiteral(final String s) public static String nullLiteral() { - return "''"; + return "null"; } public static String functionCall(final String functionName, final List args) diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index 3cff45e36546..b87d052574f4 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -33,14 +33,15 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.TimeFormatExtractionFn; import io.druid.query.filter.AndDimFilter; -import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; import io.druid.query.filter.ExpressionDimFilter; import io.druid.query.filter.LikeDimFilter; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.SelectorDimFilter; import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; import io.druid.sql.calcite.filtration.BoundRefKey; @@ -308,13 +309,10 @@ private static DimFilter toSimpleLeafFilter( return null; } - final BoundDimFilter equalFilter = Bounds.equalTo( - new BoundRefKey( - druidExpression.getSimpleExtraction().getColumn(), - druidExpression.getSimpleExtraction().getExtractionFn(), - StringComparators.LEXICOGRAPHIC - ), - "" + final DimFilter equalFilter = new SelectorDimFilter( + druidExpression.getSimpleExtraction().getColumn(), + NullHandling.defaultStringValue(), + druidExpression.getSimpleExtraction().getExtractionFn() ); return kind == SqlKind.IS_NOT_NULL ? new NotDimFilter(equalFilter) : equalFilter; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java new file mode 100644 index 000000000000..5465e311b752 --- /dev/null +++ b/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java @@ -0,0 +1,66 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.sql.calcite.expression; + +import com.google.common.collect.Iterables; +import io.druid.java.util.common.StringUtils; +import io.druid.sql.calcite.planner.PlannerContext; +import io.druid.sql.calcite.table.RowSignature; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperator; + +public class UnaryFunctionOperatorConversion implements SqlOperatorConversion +{ + private final SqlOperator operator; + private final String druidOperator; + + public UnaryFunctionOperatorConversion(final SqlOperator operator, final String druidOperator) + { + this.operator = operator; + this.druidOperator = druidOperator; + } + + @Override + public SqlOperator calciteOperator() + { + return operator; + } + + @Override + public DruidExpression toDruidExpression( + final PlannerContext plannerContext, + final RowSignature rowSignature, + final RexNode rexNode + ) + { + return OperatorConversions.convertCall( + plannerContext, + rowSignature, + rexNode, + operands -> DruidExpression.fromExpression( + StringUtils.format( + "%s(%s)", + druidOperator, + Iterables.getOnlyElement(operands).getExpression() + ) + ) + ); + } +} diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java index 216d7bcbcb14..5d4bd4b9fbdc 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java @@ -19,7 +19,6 @@ package io.druid.sql.calcite.expression.builtin; -import com.google.common.collect.ImmutableList; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.sql.calcite.expression.DruidExpression; @@ -35,6 +34,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import java.util.Arrays; import java.util.stream.Collectors; public class CeilOperatorConversion implements SqlOperatorConversion @@ -80,7 +80,7 @@ public DruidExpression toDruidExpression( // So there is no simple extraction for this operator. return DruidExpression.fromFunctionCall( "timestamp_ceil", - ImmutableList.of( + Arrays.asList( druidExpression.getExpression(), DruidExpression.stringLiteral(granularity.getPeriod().toString()), DruidExpression.numberLiteral( diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java index ac566a686059..ed6a94376547 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java @@ -19,6 +19,7 @@ package io.druid.sql.calcite.planner; +import com.google.common.base.Preconditions; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; import io.druid.java.util.common.DateTimes; @@ -104,6 +105,7 @@ public static SchemaPlus createRootSchema(final Schema druidSchema, final Author public static String escapeStringLiteral(final String s) { + Preconditions.checkNotNull(s); if (s == null) { return "''"; } else { diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java index b2ea2e183d76..59fab5b55201 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java @@ -37,6 +37,7 @@ import io.druid.sql.calcite.expression.BinaryOperatorConversion; import io.druid.sql.calcite.expression.DirectOperatorConversion; import io.druid.sql.calcite.expression.SqlOperatorConversion; +import io.druid.sql.calcite.expression.UnaryFunctionOperatorConversion; import io.druid.sql.calcite.expression.UnaryPrefixOperatorConversion; import io.druid.sql.calcite.expression.UnarySuffixOperatorConversion; import io.druid.sql.calcite.expression.builtin.BTrimOperatorConversion; @@ -117,8 +118,8 @@ public class DruidOperatorTable implements SqlOperatorTable .add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper")) .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!")) .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-")) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NULL, "== ''")) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "!= ''")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull")) .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_FALSE, "<= 0")) // Matches Evals.asBoolean .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_TRUE, "<= 0")) // Matches Evals.asBoolean .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_TRUE, "> 0")) // Matches Evals.asBoolean diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java index 5d6abfc1f01a..f4b89e6d2c8d 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java @@ -28,6 +28,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.ResourceLimitExceededException; +import io.druid.segment.DimensionHandlerUtils; import io.druid.sql.calcite.planner.PlannerContext; import org.apache.calcite.interpreter.BindableConvention; import org.apache.calcite.plan.RelOptCluster; @@ -294,7 +295,11 @@ public List accumulate(final List theConditions, final Object[ for (int i : rightKeys) { final Object value = row[i]; - final String stringValue = value != null ? String.valueOf(value) : ""; + if (value == null) { + // NULLS are not supposed to match NULLs in a join. So ignore them. + continue; + } + final String stringValue = DimensionHandlerUtils.convertObjectToString(value); values.add(stringValue); if (values.size() > maxSemiJoinRowsInMemory) { throw new ResourceLimitExceededException( @@ -308,16 +313,18 @@ public List accumulate(final List theConditions, final Object[ for (int i = 0; i < values.size(); i++) { final String value = values.get(i); - subConditions.add( - getCluster().getRexBuilder().makeCall( - SqlStdOperatorTable.EQUALS, - leftExpressions.get(i), - getCluster().getRexBuilder().makeLiteral(value) - ) - ); + // NULLS are not supposed to match NULLs in a join. So ignore them. + if (value != null) { + subConditions.add( + getCluster().getRexBuilder().makeCall( + SqlStdOperatorTable.EQUALS, + leftExpressions.get(i), + getCluster().getRexBuilder().makeLiteral(value) + ) + ); + } + theConditions.add(makeAnd(subConditions)); } - - theConditions.add(makeAnd(subConditions)); } return theConditions; } diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java index 3c063f82b36e..bc82a7f9dec4 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -46,6 +45,7 @@ import io.druid.query.topn.TopNQuery; import io.druid.query.topn.TopNResultValue; import io.druid.segment.DimensionHandlerUtils; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.server.QueryLifecycleFactory; import io.druid.server.security.AuthenticationResult; @@ -400,7 +400,7 @@ private Object coerce(final Object value, final SqlTypeName sqlType) if (SqlTypeName.CHAR_TYPES.contains(sqlType)) { if (value == null || value instanceof String) { - coercedValue = Strings.nullToEmpty((String) value); + coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); } else if (value instanceof NlsString) { coercedValue = ((NlsString) value).getValue(); } else if (value instanceof Number) { diff --git a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java index f225a8d2f83c..a7b423515c0c 100644 --- a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprMacroTable; import io.druid.server.security.AllowAllAuthenticator; @@ -143,11 +144,11 @@ public void testSelectAllInFirstFrame() throws Exception true, Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, - new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f}, + new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", NullHandling.defaultStringValue(), 2.0f}, new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, - new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", "", 6.0f} + new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", NullHandling.defaultStringValue(), 6.0f} ) ), frame @@ -170,7 +171,7 @@ public void testSelectSplitOverTwoFrames() throws Exception false, Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, - new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f} + new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", NullHandling.defaultStringValue(), 2.0f} ) ), frame @@ -187,7 +188,7 @@ public void testSelectSplitOverTwoFrames() throws Exception new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, - new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", "", 6.0f} + new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", NullHandling.defaultStringValue(), 6.0f} ) ), frame diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 52967f9d9178..95fd0fa3843b 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.hll.HLLCV1; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -87,6 +88,7 @@ import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.ForbiddenException; +import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.filtration.Filtration; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.DruidOperatorTable; @@ -476,6 +478,7 @@ public void testExplainInformationSchemaColumns() throws Exception @Test public void testSelectStar() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT * FROM druid.foo", ImmutableList.of( @@ -489,11 +492,11 @@ public void testSelectStar() throws Exception ), ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2f, 2.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -520,7 +523,7 @@ public void testSelectStarOnForbiddenTable() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, 0.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, NullHandling.defaultDoubleValue(), HLLCV1.class.getName()} ) ); } @@ -562,6 +565,8 @@ public void testExplainSelectStar() throws Exception @Test public void testSelectStarWithLimit() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; + testQuery( "SELECT * FROM druid.foo LIMIT 2", ImmutableList.of( @@ -576,7 +581,7 @@ public void testSelectStarWithLimit() throws Exception ), ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2.0f, 2.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2.0f, 2.0, HLLCV1.class.getName()} ) ); } @@ -584,6 +589,7 @@ public void testSelectStarWithLimit() throws Exception @Test public void testSelectWithProjection() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT SUBSTRING(dim2, 1, 1) FROM druid.foo LIMIT 2", ImmutableList.of( @@ -601,7 +607,7 @@ public void testSelectWithProjection() throws Exception ), ImmutableList.of( new Object[]{"a"}, - new Object[]{""} + new Object[]{nullValue} ) ); } @@ -609,6 +615,8 @@ public void testSelectWithProjection() throws Exception @Test public void testSelectStarWithLimitTimeDescending() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; + testQuery( "SELECT * FROM druid.foo ORDER BY __time DESC LIMIT 2", ImmutableList.of( @@ -624,7 +632,7 @@ public void testSelectStarWithLimitTimeDescending() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6d, HLLCV1.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6d, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} ) ); @@ -633,6 +641,7 @@ public void testSelectStarWithLimitTimeDescending() throws Exception @Test public void testSelectStarWithoutLimitTimeAscending() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT * FROM druid.foo ORDER BY __time", ImmutableList.of( @@ -665,11 +674,11 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception ), ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2f, 2.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -677,6 +686,7 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception @Test public void testSelectSingleColumnTwice() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT dim2 x, dim2 y FROM druid.foo LIMIT 2", ImmutableList.of( @@ -691,7 +701,7 @@ public void testSelectSingleColumnTwice() throws Exception ), ImmutableList.of( new Object[]{"a", "a"}, - new Object[]{"", ""} + new Object[]{nullValue, nullValue} ) ); } @@ -797,9 +807,12 @@ public void testSelfJoinWithFallback() throws Exception @Test public void testExplainSelfJoinWithFallback() throws Exception { + String emptyStringEq = NullHandling.replaceWithDefault() ? null : "\"\""; final String explanation = "BindableJoin(condition=[=($0, $2)], joinType=[inner])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING}])\n" + + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":" + + emptyStringEq + + ",\"extractionFn\":null}},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"dim1\",\"dim2\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING, dim2:STRING}])\n"; testQuery( @@ -1146,9 +1159,14 @@ public void testHavingOnApproximateCountDistinct() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", 3L}, new Object[]{"a", 2L} + ) : + ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"a", 2L} ) ); } @@ -1198,9 +1216,14 @@ public void testHavingOnExactCountDistinct() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", 3L}, new Object[]{"a", 2L} + ) : + ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"a", 2L} ) ); } @@ -1316,6 +1339,7 @@ public void testHavingOnRatio() throws Exception @Test public void testGroupByWithSelectProjections() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1335,10 +1359,10 @@ public void testGroupByWithSelectProjections() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", ""}, + new Object[]{"", nullValue}, + new Object[]{"1", nullValue}, new Object[]{"10.1", "0.1"}, - new Object[]{"2", ""}, + new Object[]{"2", nullValue}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"} ) @@ -1348,6 +1372,7 @@ public void testGroupByWithSelectProjections() throws Exception @Test public void testGroupByWithSelectAndOrderByProjections() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1387,9 +1412,9 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception new Object[]{"10.1", "0.1"}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"}, - new Object[]{"1", ""}, - new Object[]{"2", ""}, - new Object[]{"", ""} + new Object[]{"1", nullValue}, + new Object[]{"2", nullValue}, + new Object[]{"", nullValue} ) ); } @@ -1397,6 +1422,8 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception @Test public void testTopNWithSelectProjections() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; + testQuery( "SELECT\n" + " dim1," @@ -1419,10 +1446,10 @@ public void testTopNWithSelectProjections() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", ""}, + new Object[]{"", nullValue}, + new Object[]{"1", nullValue}, new Object[]{"10.1", "0.1"}, - new Object[]{"2", ""}, + new Object[]{"2", nullValue}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"} ) @@ -1432,6 +1459,8 @@ public void testTopNWithSelectProjections() throws Exception @Test public void testTopNWithSelectAndOrderByProjections() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; + testQuery( "SELECT\n" + " dim1," @@ -1459,9 +1488,9 @@ public void testTopNWithSelectAndOrderByProjections() throws Exception new Object[]{"10.1", "0.1"}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"}, - new Object[]{"1", ""}, - new Object[]{"2", ""}, - new Object[]{"", ""} + new Object[]{"1", nullValue}, + new Object[]{"2", nullValue}, + new Object[]{"", nullValue} ) ); } @@ -1550,6 +1579,7 @@ public void testPruneDeadAggregatorsThroughHaving() throws Exception @Test public void testGroupByCaseWhen() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -1580,7 +1610,7 @@ public void testGroupByCaseWhen() throws Exception + "'match-cnt'," + "(timestamp_extract(\"__time\",'DAY','UTC') == 0)," + "'zero '," - + "'')", + + DruidExpression.nullLiteral() + ")", ValueType.STRING ) ) @@ -1590,7 +1620,7 @@ public void testGroupByCaseWhen() throws Exception .build() ), ImmutableList.of( - new Object[]{"", 2L}, + new Object[]{nullValue, 2L}, new Object[]{"match-cnt", 1L}, new Object[]{"match-m1 ", 3L} ) @@ -1598,37 +1628,87 @@ public void testGroupByCaseWhen() throws Exception } @Test - public void testNullEmptyStringEquality() throws Exception + public void testIsNullString() throws Exception { - // Doesn't conform to the SQL standard, but it's how we do it. - // This example is used in the sql.md doc. + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') IS NULL", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,isnull(\"dim2\"))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + NullHandling.replaceWithDefault() ? + // Matches everything but "abc" + new Object[]{5L} : + // match only null values + new Object[]{4L} + ) + ); + } + + @Test + public void testEmptyStringEquality() throws Exception + { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') = ''", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandling.replaceWithDefault() ? "1" : "0") + + ",(\"dim2\" == ''))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + NullHandling.replaceWithDefault() ? + // Matches everything but "abc" + new Object[]{5L} : + // match only empty string + new Object[]{1L} + ) + ); + } + + @Test + public void testNullStringEquality() throws Exception + { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') = null", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandling.replaceWithDefault() ? "1" : "0") + + ",(\"dim2\" == null))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + NullHandling.replaceWithDefault() ? + // Matches everything but "abc" + ImmutableList.of(new Object[]{5L}) : + // null is not eqaual to null or any other value + ImmutableList.of() + ); - final ImmutableList wheres = ImmutableList.of( - "NULLIF(dim2, 'a') = ''", - "NULLIF(dim2, 'a') IS NULL" - ); - - for (String where : wheres) { - testQuery( - "SELECT COUNT(*)\n" - + "FROM druid.foo\n" - + "WHERE " + where, - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == ''))")) - .aggregators(AGGS(new CountAggregatorFactory("a0"))) - .context(TIMESERIES_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - // Matches everything but "abc" - new Object[]{5L} - ) - ); - } } @Test @@ -1647,7 +1727,7 @@ public void testCoalesceColumns() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "case_searched((\"dim2\" != ''),\"dim2\",\"dim1\")", + "case_searched(notnull(\"dim2\"),\"dim2\",\"dim1\")", ValueType.STRING ) ) @@ -1656,11 +1736,18 @@ public void testCoalesceColumns() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, new Object[]{"a", 2L}, new Object[]{"abc", 2L} + ) : + ImmutableList.of( + new Object[]{"", 1L}, + new Object[]{"10.1", 1L}, + new Object[]{"a", 2L}, + new Object[]{"abc", 2L} ) ); } @@ -1684,7 +1771,7 @@ public void testColumnIsNull() throws Exception .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{NullHandling.replaceWithDefault() ? 3L : 2L} ) ); } @@ -1902,14 +1989,18 @@ public void testCountNullableColumn() throws Exception .aggregators(AGGS( new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), - NOT(SELECTOR("dim2", "", null)) + NOT(SELECTOR("dim2", null, null)) ) )) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{3L} + ) : + ImmutableList.of( + new Object[]{4L} ) ); } @@ -1928,7 +2019,9 @@ public void testCountNullableExpression() throws Exception new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), EXPRESSION_FILTER( - "(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes','') != '')" + "notnull(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes'," + + DruidExpression.nullLiteral() + + "))" ) ) )) @@ -2220,7 +2313,7 @@ public void testFilterOnStringAsNumber() throws Exception public void testSimpleAggregations() throws Exception { testQuery( - "SELECT COUNT(*), COUNT(cnt), COUNT(dim1), AVG(cnt), SUM(cnt), SUM(cnt) + MIN(cnt) + MAX(cnt) FROM druid.foo", + "SELECT COUNT(*), COUNT(cnt), COUNT(dim1), AVG(cnt), SUM(cnt), SUM(cnt) + MIN(cnt) + MAX(cnt), COUNT(dim2) FROM druid.foo", ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -2231,13 +2324,17 @@ public void testSimpleAggregations() throws Exception new CountAggregatorFactory("a0"), new FilteredAggregatorFactory( new CountAggregatorFactory("a1"), - NOT(SELECTOR("dim1", "", null)) + NOT(SELECTOR("dim1", null, null)) ), new LongSumAggregatorFactory("a2:sum", "cnt"), new CountAggregatorFactory("a2:count"), new LongSumAggregatorFactory("a3", "cnt"), new LongMinAggregatorFactory("a4", "cnt"), - new LongMaxAggregatorFactory("a5", "cnt") + new LongMaxAggregatorFactory("a5", "cnt"), + new FilteredAggregatorFactory( + new CountAggregatorFactory("a6"), + NOT(SELECTOR("dim2", null, null)) + ) ) ) .postAggregators( @@ -2254,8 +2351,12 @@ public void testSimpleAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? + ImmutableList.of( + new Object[]{6L, 6L, 5L, 1L, 6L, 8L, 3L} + ) : ImmutableList.of( - new Object[]{6L, 6L, 5L, 1L, 6L, 8L} + new Object[]{6L, 6L, 6L, 1L, 6L, 8L, 4L} ) ); } @@ -2428,7 +2529,7 @@ public void testFilteredAggregations() throws Exception new FilteredAggregatorFactory( new CountAggregatorFactory("a3"), AND( - NOT(SELECTOR("dim2", "", null)), + NOT(SELECTOR("dim2", null, null)), NOT(SELECTOR("dim1", "1", null)) ) ), @@ -2473,8 +2574,12 @@ public void testFilteredAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{1L, 5L, 1L, 2L, 5L, 5L, 2L, 1L, 5L, 1L, 5L} + ) : + ImmutableList.of( + new Object[]{1L, 5L, 1L, 3L, 5L, 5L, 2L, 1L, 5L, 1L, 5L} ) ); } @@ -2542,8 +2647,12 @@ public void testFilteredAggregationWithNotIn() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{5L, 2L} + ) : + ImmutableList.of( + new Object[]{5L, 3L} ) ); } @@ -3377,7 +3486,7 @@ public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() thr .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) .setVirtualColumns( - EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"cnt\",'P1Y','','UTC')", ValueType.LONG) + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"cnt\",'P1Y',null,'UTC')", ValueType.LONG) ) .setDimFilter( BOUND( @@ -3482,10 +3591,17 @@ public void testSelectDistinctWithLimit() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} + ) : + ImmutableList.of( + new Object[]{null}, + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} ) ); } @@ -3506,10 +3622,17 @@ public void testSelectDistinctWithSortAsOuterQuery() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} + ) : + ImmutableList.of( + new Object[]{null}, + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} ) ); } @@ -3530,7 +3653,14 @@ public void testSelectDistinctWithSortAsOuterQuery2() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? + ImmutableList.of( + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} + ) : ImmutableList.of( + new Object[]{null}, new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} @@ -3566,10 +3696,17 @@ public void testSelectDistinctWithSortAsOuterQuery4() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{""}, new Object[]{"abc"}, new Object[]{"a"} + ) : + ImmutableList.of( + new Object[]{null}, + new Object[]{"abc"}, + new Object[]{"a"}, + new Object[]{""} ) ); } @@ -3684,14 +3821,14 @@ public void testExactCountDistinct() throws Exception .setAggregatorSpecs(AGGS( new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), - NOT(SELECTOR("d0", "", null)) + NOT(SELECTOR("d0", null, null)) ) )) .setContext(QUERY_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{2L} + new Object[]{NullHandling.replaceWithDefault() ? 2L : 3L} ) ); } @@ -3763,16 +3900,23 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators() throws Excep new LongSumAggregatorFactory("_a0", "a0"), new FilteredAggregatorFactory( new CountAggregatorFactory("_a1"), - NOT(SELECTOR("d0", "", null)) + NOT(SELECTOR("d0", null, null)) ) )) .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", 3L, 3L}, new Object[]{"a", 2L, 1L}, new Object[]{"abc", 1L, 1L} + ) : + ImmutableList.of( + new Object[]{null, 2L, 2L}, + new Object[]{"", 1L, 1L}, + new Object[]{"a", 2L, 2L}, + new Object[]{"abc", 1L, 1L} ) ); } @@ -3844,8 +3988,12 @@ public void testApproxCountDistinct() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{6L, 3L, 2L, 2L, 2L, 6L} + ) : + ImmutableList.of( + new Object[]{6L, 3L, 2L, 1L, 1L, 6L} ) ); } @@ -3890,7 +4038,7 @@ public void testNestedGroupBy() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "_d0:v", - "timestamp_floor(\"a0\",'PT1H','','UTC')", + "timestamp_floor(\"a0\",'PT1H',null,'UTC')", ValueType.LONG ) ) @@ -3958,8 +4106,12 @@ public void testDoubleNestedGroupBy() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{6L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 4L} ) ); } @@ -4024,8 +4176,12 @@ public void testExactCountDistinctUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{6L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 4L} ) ); } @@ -4033,6 +4189,9 @@ public void testExactCountDistinctUsingSubquery() throws Exception @Test public void testTopNFilterJoin() throws Exception { + DimFilter filter = NullHandling.replaceWithDefault() ? + IN("dim2", Arrays.asList(null, "a"), null) + : SELECTOR("dim2", "a", null); // Filters on top N values of some dimension by using an inner join. testQuery( "SELECT t1.dim1, SUM(t1.cnt)\n" @@ -4063,7 +4222,7 @@ public void testTopNFilterJoin() throws Exception .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) - .setDimFilter(IN("dim2", ImmutableList.of("", "a"), null)) + .setDimFilter(filter) .setDimensions(DIMS(new DefaultDimensionSpec("dim1", "d0"))) .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( @@ -4081,12 +4240,17 @@ public void testTopNFilterJoin() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", 1L}, new Object[]{"1", 1L}, new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, new Object[]{"abc", 1L} + ) : + ImmutableList.of( + new Object[]{"", 1L}, + new Object[]{"1", 1L} ) ); } @@ -4267,7 +4431,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception final String explanation = "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"skipEmptyBuckets\":true,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"}}], signature=[{a0:LONG}])\n" + " DruidSemiJoin(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], leftExpressions=[[SUBSTRING($3, 1, 1)]], rightKeys=[[0]])\n" - + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{d0:STRING}])\n"; + + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{d0:STRING}])\n"; testQuery( "EXPLAIN PLAN FOR SELECT COUNT(*)\n" @@ -4275,7 +4439,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception + " SELECT DISTINCT dim2\n" + " FROM druid.foo\n" + " WHERE SUBSTRING(dim2, 1, 1) IN (\n" - + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 <> ''\n" + + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n" + " )\n" + ")", ImmutableList.of(), @@ -4316,8 +4480,51 @@ public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Except .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? + ImmutableList.of( + new Object[]{3L, 2L} + ) : + ImmutableList.of( + new Object[]{5L, 3L} + ) + ); + + testQuery( + "SELECT\n" + + " SUM(cnt),\n" + + " COUNT(*)\n" + + "FROM (SELECT dim2, SUM(cnt) AS cnt FROM druid.foo GROUP BY dim2)\n" + + "WHERE dim2 IS NOT NULL", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource( + new QueryDataSource( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(QSS(Filtration.eternity())) + .setDimFilter(NOT(SELECTOR("dim2", null, null))) + .setGranularity(Granularities.ALL) + .setDimensions(DIMS(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ) + ) + .setInterval(QSS(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setAggregatorSpecs(AGGS( + new LongSumAggregatorFactory("_a0", "a0"), + new CountAggregatorFactory("_a1") + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{3L, 2L} + ) : + ImmutableList.of( + new Object[]{4L, 3L} ) ); } @@ -4356,8 +4563,12 @@ public void testExactCountDistinctUsingSubqueryWithWhereToOuterFilter() throws E .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{3L, 1L} + ) : + ImmutableList.of( + new Object[]{2L, 1L} ) ); } @@ -4443,10 +4654,15 @@ public void testHistogramUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L}, new Object[]{"3", 1L} + ) : + ImmutableList.of( + new Object[]{"1", 2L}, + new Object[]{"2", 2L} ) ); } @@ -4493,9 +4709,14 @@ public void testHistogramUsingSubqueryWithSort() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L} + ) : + ImmutableList.of( + new Object[]{"1", 2L}, + new Object[]{"2", 2L} ) ); } @@ -4649,6 +4870,7 @@ public void testSillyQuarters() throws Exception @Test public void testRegexpExtract() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT DISTINCT\n" + " REGEXP_EXTRACT(dim1, '^.'),\n" @@ -4685,7 +4907,7 @@ public void testRegexpExtract() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, + new Object[]{nullValue, nullValue}, new Object[]{"1", "1"}, new Object[]{"2", "2"}, new Object[]{"a", "a"}, @@ -4697,6 +4919,7 @@ public void testRegexpExtract() throws Exception @Test public void testGroupBySortPushDown() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT dim2, dim1, SUM(cnt) FROM druid.foo GROUP BY dim2, dim1 ORDER BY dim1 LIMIT 4", ImmutableList.of( @@ -4729,7 +4952,7 @@ public void testGroupBySortPushDown() throws Exception ImmutableList.of( new Object[]{"a", "", 1L}, new Object[]{"a", "1", 1L}, - new Object[]{"", "10.1", 1L}, + new Object[]{nullValue, "10.1", 1L}, new Object[]{"", "2", 1L} ) ); @@ -4738,6 +4961,7 @@ public void testGroupBySortPushDown() throws Exception @Test public void testGroupByLimitPushDownWithHavingOnLong() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT dim1, dim2, SUM(cnt) AS thecnt " + "FROM druid.foo " @@ -4773,11 +4997,18 @@ public void testGroupByLimitPushDownWithHavingOnLong() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"10.1", "", 1L}, new Object[]{"2", "", 1L}, new Object[]{"abc", "", 1L}, new Object[]{"", "a", 1L} + ) : + ImmutableList.of( + new Object[]{"10.1", null, 1L}, + new Object[]{"abc", null, 1L}, + new Object[]{"2", "", 1L}, + new Object[]{"", "a", 1L} ) ); } @@ -5157,7 +5388,7 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(\"__time\",'P1Y','','UTC')", + "timestamp_floor(\"__time\",'P1Y',null,'UTC')", ValueType.LONG ) ) @@ -5197,12 +5428,21 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{T("2000"), "", 2L}, new Object[]{T("2000"), "a", 1L}, new Object[]{T("2001"), "", 1L}, new Object[]{T("2001"), "a", 1L}, new Object[]{T("2001"), "abc", 1L} + ) : + ImmutableList.of( + new Object[]{T("2000"), null, 1L}, + new Object[]{T("2000"), "", 1L}, + new Object[]{T("2000"), "a", 1L}, + new Object[]{T("2001"), null, 1L}, + new Object[]{T("2001"), "a", 1L}, + new Object[]{T("2001"), "abc", 1L} ) ); } @@ -5235,6 +5475,7 @@ public void testGroupByStringLength() throws Exception @Test public void testFilterAndGroupByLookup() throws Exception { + String nullValue = NullHandling.replaceWithDefault() ? "" : null; final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn( null, "lookyloo", @@ -5279,7 +5520,7 @@ public void testFilterAndGroupByLookup() throws Exception .build() ), ImmutableList.of( - new Object[]{"", 5L}, + new Object[]{nullValue, 5L}, new Object[]{"xabc", 1L} ) ); @@ -5317,7 +5558,7 @@ public void testCountDistinctOfLookup() throws Exception .build() ), ImmutableList.of( - new Object[]{2L} + new Object[]{NullHandling.replaceWithDefault() ? 2L : 1L} ) ); } @@ -5476,7 +5717,7 @@ public void testTimeseriesUsingTimeFloorWithTimeShift() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(timestamp_shift(\"__time\",'P1D',-1),'P1M','','UTC')", + "timestamp_floor(timestamp_shift(\"__time\",'P1D',-1),'P1M',null,'UTC')", ValueType.LONG ) ) @@ -5524,7 +5765,7 @@ public void testTimeseriesUsingTimeFloorWithTimestampAdd() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor((\"__time\" + -86400000),'P1M','','UTC')", + "timestamp_floor((\"__time\" + -86400000),'P1M',null,'UTC')", ValueType.LONG ) ) @@ -5651,7 +5892,7 @@ public void testTimeseriesLosAngelesUsingTimeFloorConnectionLosAngeles() throws public void testTimeseriesDontSkipEmptyBuckets() throws Exception { // Tests that query context parameters are passed through to the underlying query engine. - + Long defaultVal = NullHandling.replaceWithDefault() ? 0L : null; testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS, @@ -5673,29 +5914,29 @@ public void testTimeseriesDontSkipEmptyBuckets() throws Exception ), ImmutableList.builder() .add(new Object[]{1L, T("2000-01-01")}) - .add(new Object[]{0L, T("2000-01-01T01")}) - .add(new Object[]{0L, T("2000-01-01T02")}) - .add(new Object[]{0L, T("2000-01-01T03")}) - .add(new Object[]{0L, T("2000-01-01T04")}) - .add(new Object[]{0L, T("2000-01-01T05")}) - .add(new Object[]{0L, T("2000-01-01T06")}) - .add(new Object[]{0L, T("2000-01-01T07")}) - .add(new Object[]{0L, T("2000-01-01T08")}) - .add(new Object[]{0L, T("2000-01-01T09")}) - .add(new Object[]{0L, T("2000-01-01T10")}) - .add(new Object[]{0L, T("2000-01-01T11")}) - .add(new Object[]{0L, T("2000-01-01T12")}) - .add(new Object[]{0L, T("2000-01-01T13")}) - .add(new Object[]{0L, T("2000-01-01T14")}) - .add(new Object[]{0L, T("2000-01-01T15")}) - .add(new Object[]{0L, T("2000-01-01T16")}) - .add(new Object[]{0L, T("2000-01-01T17")}) - .add(new Object[]{0L, T("2000-01-01T18")}) - .add(new Object[]{0L, T("2000-01-01T19")}) - .add(new Object[]{0L, T("2000-01-01T20")}) - .add(new Object[]{0L, T("2000-01-01T21")}) - .add(new Object[]{0L, T("2000-01-01T22")}) - .add(new Object[]{0L, T("2000-01-01T23")}) + .add(new Object[]{defaultVal, T("2000-01-01T01")}) + .add(new Object[]{defaultVal, T("2000-01-01T02")}) + .add(new Object[]{defaultVal, T("2000-01-01T03")}) + .add(new Object[]{defaultVal, T("2000-01-01T04")}) + .add(new Object[]{defaultVal, T("2000-01-01T05")}) + .add(new Object[]{defaultVal, T("2000-01-01T06")}) + .add(new Object[]{defaultVal, T("2000-01-01T07")}) + .add(new Object[]{defaultVal, T("2000-01-01T08")}) + .add(new Object[]{defaultVal, T("2000-01-01T09")}) + .add(new Object[]{defaultVal, T("2000-01-01T10")}) + .add(new Object[]{defaultVal, T("2000-01-01T11")}) + .add(new Object[]{defaultVal, T("2000-01-01T12")}) + .add(new Object[]{defaultVal, T("2000-01-01T13")}) + .add(new Object[]{defaultVal, T("2000-01-01T14")}) + .add(new Object[]{defaultVal, T("2000-01-01T15")}) + .add(new Object[]{defaultVal, T("2000-01-01T16")}) + .add(new Object[]{defaultVal, T("2000-01-01T17")}) + .add(new Object[]{defaultVal, T("2000-01-01T18")}) + .add(new Object[]{defaultVal, T("2000-01-01T19")}) + .add(new Object[]{defaultVal, T("2000-01-01T20")}) + .add(new Object[]{defaultVal, T("2000-01-01T21")}) + .add(new Object[]{defaultVal, T("2000-01-01T22")}) + .add(new Object[]{defaultVal, T("2000-01-01T23")}) .build() ); } @@ -5891,7 +6132,7 @@ public void testGroupByExtractFloorTime() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_extract(timestamp_floor(\"__time\",'P1Y','','UTC'),'YEAR','UTC')", + "timestamp_extract(timestamp_floor(\"__time\",'P1Y',null,'UTC'),'YEAR','UTC')", ValueType.LONG ) ) @@ -5926,7 +6167,7 @@ public void testGroupByExtractFloorTimeLosAngeles() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_extract(timestamp_floor(\"__time\",'P1Y','','America/Los_Angeles'),'YEAR','America/Los_Angeles')", + "timestamp_extract(timestamp_floor(\"__time\",'P1Y',null,'America/Los_Angeles'),'YEAR','America/Los_Angeles')", ValueType.LONG ) ) @@ -5965,7 +6206,7 @@ public void testTimeseriesWithLimitNoTopN() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(\"__time\",'P1M','','UTC')", + "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG ) ) @@ -6009,7 +6250,7 @@ public void testTimeseriesWithLimit() throws Exception .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .virtualColumns( - EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M','','UTC')", ValueType.LONG) + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG) ) .dimension(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG)) .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) @@ -6042,7 +6283,7 @@ public void testTimeseriesWithOrderByAndLimit() throws Exception .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .virtualColumns( - EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M','','UTC')", ValueType.LONG) + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG) ) .dimension(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG)) .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) @@ -6073,7 +6314,7 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d1:v", - "timestamp_floor(\"__time\",'P1M','','UTC')", + "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG ) ) @@ -6100,12 +6341,21 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", T("2000-01-01"), 2L}, new Object[]{"", T("2001-01-01"), 1L}, new Object[]{"a", T("2000-01-01"), 1L}, new Object[]{"a", T("2001-01-01"), 1L}, new Object[]{"abc", T("2001-01-01"), 1L} + ) : + ImmutableList.of( + new Object[]{null, T("2000-01-01"), 1L}, + new Object[]{null, T("2001-01-01"), 1L}, + new Object[]{"", T("2000-01-01"), 1L}, + new Object[]{"a", T("2000-01-01"), 1L}, + new Object[]{"a", T("2001-01-01"), 1L}, + new Object[]{"abc", T("2001-01-01"), 1L} ) ); } @@ -6235,7 +6485,13 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) - .filters(AND(SELECTOR("dim1", "def", null), SELECTOR("dim2", "abc", null))) + .filters(OR( + SELECTOR("dim1", "def", null), + AND( + SELECTOR("dim1", "def", null), + SELECTOR("dim2", "abc", null) + ) + )) .columns("__time", "cnt", "dim1", "dim2") .resultFormat(ScanQuery.RESULT_FORMAT_COMPACTED_LIST) .context(QUERY_CONTEXT_DEFAULT) @@ -6250,8 +6506,9 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception @Test public void testUsingSubqueryAsFilterWithInnerSort() throws Exception { - // Regression test for https://github.com/druid-io/druid/issues/4208 + String nullValue = NullHandling.replaceWithDefault() ? "" : null; + // Regression test for https://github.com/druid-io/druid/issues/4208 testQuery( "SELECT dim1, dim2 FROM druid.foo\n" + " WHERE dim2 IN (\n" @@ -6288,13 +6545,20 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", "a"}, - new Object[]{"10.1", ""}, + new Object[]{"10.1", nullValue}, new Object[]{"2", ""}, new Object[]{"1", "a"}, new Object[]{"def", "abc"}, - new Object[]{"abc", ""} + new Object[]{"abc", nullValue} + ) : + ImmutableList.of( + new Object[]{"", "a"}, + new Object[]{"2", ""}, + new Object[]{"1", "a"}, + new Object[]{"def", "abc"} ) ); } diff --git a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java index b565484055ce..e2e7ce948072 100644 --- a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; @@ -194,8 +195,8 @@ public void testStrpos() rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.VARCHAR)), rexBuilder.makeLiteral("ax") ), - DruidExpression.fromExpression("(strpos('','ax') + 1)"), - 0L + DruidExpression.fromExpression("(strpos(null,'ax') + 1)"), + NullHandling.replaceWithDefault() ? 0L : null ); } @@ -328,7 +329,7 @@ public void testDateTrunc() rexBuilder.makeLiteral("hour"), timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")) ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H',null,'UTC')"), DateTimes.of("2000-02-03T04:00:00").getMillis() ); @@ -338,7 +339,7 @@ public void testDateTrunc() rexBuilder.makeLiteral("DAY"), timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")) ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'P1D','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'P1D',null,'UTC')"), DateTimes.of("2000-02-03T00:00:00").getMillis() ); } @@ -389,7 +390,7 @@ public void testTimeFloor() timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")), rexBuilder.makeLiteral("PT1H") ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H',null,'UTC')"), DateTimes.of("2000-02-03T04:00:00").getMillis() ); @@ -401,7 +402,7 @@ public void testTimeFloor() rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.TIMESTAMP)), rexBuilder.makeLiteral("America/Los_Angeles") ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D','','America/Los_Angeles')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'America/Los_Angeles')"), DateTimes.of("2000-02-02T08:00:00").getMillis() ); } @@ -417,7 +418,7 @@ public void testOtherTimeFloor() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1Y','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1Y',null,'UTC')"), DateTimes.of("2000").getMillis() ); } @@ -433,7 +434,7 @@ public void testOtherTimeCeil() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y','','UTC')"), + DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y',null,'UTC')"), DateTimes.of("2001").getMillis() ); } @@ -668,7 +669,7 @@ public void testCastAsTimestamp() ), DruidExpression.of( null, - "timestamp_parse(\"tstr\",'','UTC')" + "timestamp_parse(\"tstr\",null,'UTC')" ), DateTimes.of("2000-02-03T04:05:06Z").getMillis() ); @@ -715,7 +716,7 @@ public void testCastAsDate() typeFactory.createSqlType(SqlTypeName.DATE), inputRef("t") ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'UTC')"), DateTimes.of("2000-02-03").getMillis() ); @@ -725,7 +726,7 @@ public void testCastAsDate() inputRef("dstr") ), DruidExpression.fromExpression( - "timestamp_floor(timestamp_parse(\"dstr\",'','UTC'),'P1D','','UTC')" + "timestamp_floor(timestamp_parse(\"dstr\",null,'UTC'),'P1D',null,'UTC')" ), DateTimes.of("2000-02-03").getMillis() ); @@ -743,7 +744,7 @@ public void testCastFromDate() ) ), DruidExpression.fromExpression( - "timestamp_format(timestamp_floor(\"t\",'P1D','','UTC'),'yyyy-MM-dd','UTC')" + "timestamp_format(timestamp_floor(\"t\",'P1D',null,'UTC'),'yyyy-MM-dd','UTC')" ), "2000-02-03" ); @@ -756,7 +757,7 @@ public void testCastFromDate() inputRef("t") ) ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'UTC')"), DateTimes.of("2000-02-03").getMillis() ); } diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index 640d5e752e3f..f68714804fa6 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -23,12 +23,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; import io.druid.math.expr.ExprMacroTable; import io.druid.query.QueryInterruptedException; import io.druid.query.ResourceLimitExceededException; +import io.druid.common.config.NullHandling; import io.druid.server.security.AllowAllAuthenticator; import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthTestUtils; @@ -217,10 +219,18 @@ public void testFieldAliasingGroupBy() throws Exception ).rhs; Assert.assertEquals( + NullHandling.replaceWithDefault() ? ImmutableList.of( ImmutableMap.of("x", "", "y", ""), ImmutableMap.of("x", "a", "y", "a"), ImmutableMap.of("x", "abc", "y", "abc") + ) : + ImmutableList.of( + // x and y both should be null instead of empty string + Maps.transformValues(ImmutableMap.of("x", "", "y", ""), (val) -> null), + ImmutableMap.of("x", "", "y", ""), + ImmutableMap.of("x", "a", "y", "a"), + ImmutableMap.of("x", "abc", "y", "abc") ), rows ); diff --git a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java index 937ba7f81096..9dc6fe72aeed 100644 --- a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java @@ -29,7 +29,6 @@ public class CalcitesTest extends CalciteTestBase @Test public void testEscapeStringLiteral() { - Assert.assertEquals("''", Calcites.escapeStringLiteral(null)); Assert.assertEquals("''", Calcites.escapeStringLiteral("")); Assert.assertEquals("'foo'", Calcites.escapeStringLiteral("foo")); Assert.assertEquals("'foo bar'", Calcites.escapeStringLiteral("foo bar")); From fd570b32355e8c5520ec5fc9e7e7bd7c0b3ad6a1 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 1 Mar 2018 01:48:17 +0530 Subject: [PATCH 05/38] fix test fix test failures more test fixes --- .../main/java/io/druid/math/expr/Expr.java | 3 ++ .../java/io/druid/math/expr/ExprEval.java | 5 ++- .../java/io/druid/math/expr/EvalTest.java | 6 +++- .../druid/query/expression/TrimExprMacro.java | 2 +- .../DoubleGroupByColumnSelectorStrategy.java | 34 +++++++++++++++---- .../FloatGroupByColumnSelectorStrategy.java | 34 +++++++++++++++---- .../LongGroupByColumnSelectorStrategy.java | 32 +++++++++++++---- .../segment/filter/ExpressionFilter.java | 3 ++ .../druid/sql/calcite/CalciteQueryTest.java | 22 +++++++----- 9 files changed, 108 insertions(+), 33 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index ecde03215d3c..690146788eae 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -365,6 +365,9 @@ public ExprEval eval(ObjectBinding bindings) // Result of any Binary expressions is null if any of the argument is null. // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec. + if (NullHandling.sqlCompatible() && (leftVal.value() == null || rightVal.value() == null)) { + return ExprEval.of(null); + } if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) { return evalString(leftVal.asString(), rightVal.asString()); diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index a1e4306f4fd1..5809b6264d27 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -19,7 +19,6 @@ package io.druid.math.expr; -import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; @@ -155,7 +154,7 @@ private static class DoubleExprEval extends NumericExprEval { private DoubleExprEval(Number value) { - super(Preconditions.checkNotNull(value, "value")); + super(value); } @Override @@ -195,7 +194,7 @@ private static class LongExprEval extends NumericExprEval { private LongExprEval(Number value) { - super(Preconditions.checkNotNull(value, "value")); + super(value); } @Override diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 01c854be572a..7910f6a94913 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -20,6 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -139,7 +140,10 @@ public void testLongEval() Assert.assertEquals(1271055781L, evalLong("unix_timestamp('2010-04-12T07:03:01')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + Assert.assertEquals( + NullHandling.replaceWithDefault() ? "NULL" : "", + eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString() + ); Assert.assertEquals("x", eval("nvl(if(x == 9223372036854775806, '', 'x'), 'NULL')", bindings).asString()); } diff --git a/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java b/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java index b026f267baa2..be1c580221b4 100644 --- a/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java @@ -113,7 +113,7 @@ public ExprEval eval(final ObjectBinding bindings) { final ExprEval stringEval = stringExpr.eval(bindings); - if (chars.length == 0 || stringEval.isNull()) { + if (chars.length == 0 || stringEval.value() == null) { return stringEval; } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java index ae0250c6f929..dff778a0e19b 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java @@ -20,8 +20,10 @@ package io.druid.query.groupby.epinephelinae.column; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Map; @@ -30,7 +32,7 @@ public class DoubleGroupByColumnSelectorStrategy implements GroupByColumnSelecto @Override public int getGroupingKeySize() { - return Double.BYTES; + return Double.BYTES + Byte.BYTES; } @Override @@ -38,26 +40,44 @@ public void processValueFromGroupingKey( GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap ) { - final double val = key.getDouble(selectorPlus.getKeyBufferPosition()); - resultMap.put(selectorPlus.getOutputName(), val); + if (key.get(selectorPlus.getKeyBufferPosition()) == (byte) 1) { + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultDoubleValue()); + } else { + final double val = key.getDouble(selectorPlus.getKeyBufferPosition() + Byte.BYTES); + resultMap.put(selectorPlus.getOutputName(), val); + } } @Override public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] values) { - values[columnIndex] = selector.getDouble(); + if (NullHandling.sqlCompatible() && selector.isNull()) { + values[columnIndex] = null; + } else { + values[columnIndex] = selector.getDouble(); + } } @Override + @Nullable public Object getOnlyValue(ColumnValueSelector selector) { + if (NullHandling.sqlCompatible() && selector.isNull()) { + return null; + } return selector.getDouble(); } @Override - public void writeToKeyBuffer(int keyBufferPosition, Object obj, ByteBuffer keyBuffer) + public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBuffer keyBuffer) { - keyBuffer.putDouble(keyBufferPosition, (Double) obj); + if (obj == null) { + keyBuffer.put(keyBufferPosition, (byte) 1); + keyBuffer.putDouble(keyBufferPosition + Byte.BYTES, 0.0d); + } else { + keyBuffer.put(keyBufferPosition, (byte) 0); + keyBuffer.putDouble(keyBufferPosition + Byte.BYTES, (Double) obj); + } } @Override @@ -65,7 +85,7 @@ public void initGroupingKeyColumnValue( int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack ) { - keyBuffer.putDouble(keyBufferPosition, (Double) rowObj); + writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); stack[columnIndex] = 1; } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java index 226e50f32291..7df4ddd2f453 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java @@ -19,8 +19,10 @@ package io.druid.query.groupby.epinephelinae.column; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Map; @@ -30,7 +32,7 @@ public class FloatGroupByColumnSelectorStrategy implements GroupByColumnSelector @Override public int getGroupingKeySize() { - return Float.BYTES; + return Float.BYTES + Byte.BYTES; } @Override @@ -38,26 +40,44 @@ public void processValueFromGroupingKey( GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap ) { - final float val = key.getFloat(selectorPlus.getKeyBufferPosition()); - resultMap.put(selectorPlus.getOutputName(), val); + if (key.get(selectorPlus.getKeyBufferPosition()) == (byte) 1) { + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultFloatValue()); + } else { + final float val = key.getFloat(selectorPlus.getKeyBufferPosition() + Byte.BYTES); + resultMap.put(selectorPlus.getOutputName(), val); + } } @Override public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] valuess) { - valuess[columnIndex] = selector.getFloat(); + if (NullHandling.sqlCompatible() && selector.isNull()) { + valuess[columnIndex] = null; + } else { + valuess[columnIndex] = selector.getFloat(); + } } @Override + @Nullable public Object getOnlyValue(ColumnValueSelector selector) { + if (NullHandling.sqlCompatible() && selector.isNull()) { + return null; + } return selector.getFloat(); } @Override - public void writeToKeyBuffer(int keyBufferPosition, Object obj, ByteBuffer keyBuffer) + public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBuffer keyBuffer) { - keyBuffer.putFloat(keyBufferPosition, (Float) obj); + if (obj == null) { + keyBuffer.put(keyBufferPosition, (byte) 1); + keyBuffer.putFloat(keyBufferPosition + Byte.BYTES, 0f); + } else { + keyBuffer.put(keyBufferPosition, (byte) 0); + keyBuffer.putFloat(keyBufferPosition + Byte.BYTES, (Float) obj); + } } @Override @@ -65,7 +85,7 @@ public void initGroupingKeyColumnValue( int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack ) { - keyBuffer.putFloat(keyBufferPosition, (Float) rowObj); + writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); stack[columnIndex] = 1; } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java index dac7ef639f0b..8c92f74e7803 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java @@ -19,8 +19,10 @@ package io.druid.query.groupby.epinephelinae.column; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Map; @@ -30,7 +32,7 @@ public class LongGroupByColumnSelectorStrategy implements GroupByColumnSelectorS @Override public int getGroupingKeySize() { - return Long.BYTES; + return Long.BYTES + Byte.BYTES; } @Override @@ -38,26 +40,44 @@ public void processValueFromGroupingKey( GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap ) { - final long val = key.getLong(selectorPlus.getKeyBufferPosition()); - resultMap.put(selectorPlus.getOutputName(), val); + if (key.get(selectorPlus.getKeyBufferPosition()) == (byte) 1) { + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultLongValue()); + } else { + final long val = key.getLong(selectorPlus.getKeyBufferPosition() + Byte.BYTES); + resultMap.put(selectorPlus.getOutputName(), val); + } } @Override public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] valuess) { - valuess[columnIndex] = selector.getLong(); + if (NullHandling.sqlCompatible() && selector.isNull()) { + valuess[columnIndex] = null; + } else { + valuess[columnIndex] = selector.getLong(); + } } @Override + @Nullable public Object getOnlyValue(ColumnValueSelector selector) { + if (NullHandling.sqlCompatible() && selector.isNull()) { + return null; + } return selector.getLong(); } @Override public void writeToKeyBuffer(int keyBufferPosition, Object obj, ByteBuffer keyBuffer) { - keyBuffer.putLong(keyBufferPosition, (Long) obj); + if (obj == null) { + keyBuffer.put(keyBufferPosition, (byte) 1); + keyBuffer.putLong(keyBufferPosition + Byte.BYTES, 0L); + } else { + keyBuffer.put(keyBufferPosition, (byte) 0); + keyBuffer.putLong(keyBufferPosition + Byte.BYTES, (Long) obj); + } } @Override @@ -65,7 +85,7 @@ public void initGroupingKeyColumnValue( int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack ) { - keyBuffer.putLong(keyBufferPosition, (Long) rowObj); + writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); stack[columnIndex] = 1; } diff --git a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java index c13d9edd3aa0..42c2b32702af 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java @@ -59,6 +59,9 @@ public ValueMatcher makeMatcher(final ColumnSelectorFactory factory) @Override public boolean matches() { + if (NullHandling.sqlCompatible() && selector.isNull()) { + return false; + } return Evals.asBoolean(selector.getLong()); } diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 95fd0fa3843b..c1d50121acfb 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -1288,9 +1288,13 @@ public void testColumnComparison() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{"", 1.0f, 1L}, new Object[]{"2", 3.0f, 1L} + ) : + ImmutableList.of( + new Object[]{"2", 3.0f, 1L} ) ); } @@ -2832,10 +2836,16 @@ public void testExpressionFilteringAndGroupingOnStringCastToNumber() throws Exce .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.replaceWithDefault() ? ImmutableList.of( new Object[]{10.0f, 1L}, new Object[]{2.0f, 1L}, new Object[]{0.0f, 4L} + ) : + ImmutableList.of( + new Object[]{10.0f, 1L}, + new Object[]{2.0f, 1L}, + new Object[]{0.0f, 1L} ) ); } @@ -3411,8 +3421,6 @@ public void testCountStarWithTimeFilterOnLongColumnUsingTimestampToMillis() thro @Test public void testSumOfString() throws Exception { - // Perhaps should be 13, but dim1 has "1", "2" and "10.1"; and CAST('10.1' AS INTEGER) = 0 since parsing is strict. - testQuery( "SELECT SUM(CAST(dim1 AS INTEGER)) FROM druid.foo", ImmutableList.of( @@ -3432,7 +3440,7 @@ public void testSumOfString() throws Exception .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{13L} ) ); } @@ -3440,8 +3448,6 @@ public void testSumOfString() throws Exception @Test public void testSumOfExtractionFn() throws Exception { - // Perhaps should be 13, but dim1 has "1", "2" and "10.1"; and CAST('10.1' AS INTEGER) = 0 since parsing is strict. - testQuery( "SELECT SUM(CAST(SUBSTRING(dim1, 1, 10) AS INTEGER)) FROM druid.foo", ImmutableList.of( @@ -3461,7 +3467,7 @@ public void testSumOfExtractionFn() throws Exception .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{13L} ) ); } @@ -5313,7 +5319,7 @@ public void testGroupByFloor() throws Exception .build() ), ImmutableList.of( - new Object[]{0.0f, 3L}, + new Object[]{NullHandling.defaultFloatValue(), 3L}, new Object[]{1.0f, 1L}, new Object[]{2.0f, 1L}, new Object[]{10.0f, 1L} @@ -5367,7 +5373,7 @@ public void testGroupByFloorWithOrderBy() throws Exception new Object[]{10.0f, 1L}, new Object[]{2.0f, 1L}, new Object[]{1.0f, 1L}, - new Object[]{0.0f, 3L} + new Object[]{NullHandling.defaultFloatValue(), 3L} ) ); } From 14a81484d4fc721517e57dd8f4796b59fa77c04b Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 1 Mar 2018 04:06:56 +0530 Subject: [PATCH 06/38] enable more tests --- .travis.yml | 20 +++++++++++++++++++ .../java/io/druid/math/expr/ExprEval.java | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 285b3895bb29..9fe69bc52571 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,17 @@ matrix: # Server module test is run without the parallel-test option because it's memory sensitive and often fails with that option. script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -pl server + # server module test with SQL Compatibility enabled + - sudo: false + env: + - NAME="server module test with SQL Compatibility enabled" + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + # Server module test is run without the parallel-test option because it's memory sensitive and often fails with that option. + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -pl server + + # other modules test - sudo: false env: @@ -61,6 +72,15 @@ matrix: - unset _JAVA_OPTIONS script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing,!server' + # other modules test with SQL Compatibility enabled + - sudo: false + env: + - NAME="other modules test with SQL Compatibility" + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -Ddruid.generic.useDefaultValueForNull=false -pl '!processing,!server' + # run integration tests - sudo: required services: diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 5809b6264d27..7bb32b12547f 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -128,7 +128,7 @@ private abstract static class NumericExprEval extends ExprEval private NumericExprEval(Number value) { - super(value); + super(value == null ? NullHandling.defaultDoubleValue() : value); } @Override From ced5c24d196f19aebf4fb4499ec6f681e4c9f183 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 1 Mar 2018 04:20:05 +0530 Subject: [PATCH 07/38] Fix ExpressionFilterTest fix format Fix test Fix tests more test fix more test failures fix formatting --- .../java/io/druid/math/expr/ExprEval.java | 2 +- ...penderatorDriverRealtimeIndexTaskTest.java | 46 +++++++++++-------- .../common/task/RealtimeIndexTaskTest.java | 39 ++++++++++------ .../druid/query/expression/TrimExprMacro.java | 4 +- .../segment/filter/ExpressionFilterTest.java | 40 ++++++++++++---- .../virtual/ExpressionVirtualColumnTest.java | 6 ++- .../druid/query/expression/ExprMacroTest.java | 28 ++++++----- 7 files changed, 109 insertions(+), 56 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 7bb32b12547f..3906fc945aeb 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -294,7 +294,7 @@ private Number asNumber() @Override public boolean isNull() { - return !NullHandling.replaceWithDefault() && asNumber() == null; + return asNumber() == null; } @Override diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index 2f1097c3e30f..786f011e9333 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -30,6 +30,7 @@ import com.google.common.util.concurrent.MoreExecutors; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; +import io.druid.common.config.NullHandling; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; @@ -354,8 +355,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -418,8 +419,8 @@ public void testLateData() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -485,8 +486,8 @@ public void testMaxRowsPerSegment() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2000, sumMetric(task, null, "rows")); - Assert.assertEquals(2000, sumMetric(task, null, "met1")); + Assert.assertEquals(2000, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(2000, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -515,7 +516,7 @@ public void testMaxRowsPerSegment() throws Exception Assert.assertEquals(TaskState.SUCCESS, taskStatus.getStatusCode()); } - @Test(timeout = 60_000L) + @Test(timeout = 60_00000L) public void testTransformSpec() throws Exception { expectPublishedSegments(2); @@ -555,10 +556,14 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); + } else { + Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); + } + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -684,8 +689,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -757,7 +762,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows")); + Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -775,7 +780,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows")); + Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); awaitHandoffs(); @@ -832,7 +837,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows")); + Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); // Trigger graceful shutdown. task1.stopGracefully(); @@ -1259,7 +1264,7 @@ public List getLocations() ); } - public long sumMetric(final Task task, final DimFilter filter, final String metric) throws Exception + public Long sumMetric(final Task task, final DimFilter filter, final String metric) throws Exception { // Do a query. TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() @@ -1275,6 +1280,11 @@ public long sumMetric(final Task task, final DimFilter filter, final String metr List> results = task.getQueryRunner(query).run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); - return results.isEmpty() ? 0 : results.get(0).getValue().getLongMetric(metric); + + if (results.isEmpty()) { + return 0L; + } else { + return results.get(0).getValue().getLongMetric(metric); + } } } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index 9f050fdcab47..44cbda829f28 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -32,6 +32,7 @@ import com.google.common.util.concurrent.MoreExecutors; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; +import io.druid.common.config.NullHandling; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; @@ -207,7 +208,8 @@ public InputRow nextRow() @Override public Runnable commit() { - return () -> {}; + return () -> { + }; } @Override @@ -353,8 +355,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -422,10 +424,15 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(1, sumMetric(task, null, "rows")); - Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); - Assert.assertEquals(1, sumMetric(task, null, "met1")); + Assert.assertEquals(1, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); + } else { + Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); + + } + Assert.assertEquals(1, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -551,8 +558,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -624,7 +631,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows")); + Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -645,7 +652,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows")); + Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -706,7 +713,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows")); + Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); // Trigger graceful shutdown. task1.stopGracefully(); @@ -1095,7 +1102,7 @@ public List getLocations() return toolboxFactory.build(task); } - public long sumMetric(final Task task, final DimFilter filter, final String metric) throws Exception + public Long sumMetric(final Task task, final DimFilter filter, final String metric) throws Exception { // Do a query. TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() @@ -1111,6 +1118,10 @@ public long sumMetric(final Task task, final DimFilter filter, final String metr List> results = task.getQueryRunner(query).run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); - return results.isEmpty() ? 0 : results.get(0).getValue().getLongMetric(metric); + if (results.isEmpty()) { + return 0L; + } else { + return results.get(0).getValue().getLongMetric(metric); + } } } diff --git a/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java b/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java index be1c580221b4..ebaf6f42dd89 100644 --- a/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TrimExprMacro.java @@ -176,13 +176,13 @@ public ExprEval eval(final ObjectBinding bindings) { final ExprEval stringEval = stringExpr.eval(bindings); - if (stringEval.isNull()) { + if (stringEval.value() == null) { return stringEval; } final ExprEval charsEval = charsExpr.eval(bindings); - if (charsEval.isNull()) { + if (charsEval.value() == null) { return stringEval; } diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index 235e65147b91..650c6256a348 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -121,8 +121,14 @@ public void testOneSingleValuedStringColumn() assertFilterMatches(EDF("dim3 == 1.0"), ImmutableList.of("3", "4", "6")); assertFilterMatches(EDF("dim3 == 1.234"), ImmutableList.of("9")); assertFilterMatches(EDF("dim3 < '2'"), ImmutableList.of("0", "1", "3", "4", "6", "9")); - assertFilterMatches(EDF("dim3 < 2"), ImmutableList.of("0", "3", "4", "6", "7", "9")); - assertFilterMatches(EDF("dim3 < 2.0"), ImmutableList.of("0", "3", "4", "6", "7", "9")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(EDF("dim3 < 2"), ImmutableList.of("0", "3", "4", "6", "7", "9")); + assertFilterMatches(EDF("dim3 < 2.0"), ImmutableList.of("0", "3", "4", "6", "7", "9")); + } else { + // Empty String and "a" will not match + assertFilterMatches(EDF("dim3 < 2"), ImmutableList.of("3", "4", "6", "9")); + assertFilterMatches(EDF("dim3 < 2.0"), ImmutableList.of("3", "4", "6", "9")); + } assertFilterMatches(EDF("like(dim3, '1%')"), ImmutableList.of("1", "3", "4", "6", "9")); } @@ -145,7 +151,12 @@ public void testOneMultiValuedStringColumn() @Test public void testOneLongColumn() { - assertFilterMatches(EDF("dim1 == ''"), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(EDF("dim1 == ''"), ImmutableList.of("0")); + } else { + // A long does not match empty string + assertFilterMatches(EDF("dim1 == ''"), ImmutableList.of()); + } assertFilterMatches(EDF("dim1 == '1'"), ImmutableList.of("1")); assertFilterMatches(EDF("dim1 == 2"), ImmutableList.of("2")); assertFilterMatches(EDF("dim1 < '2'"), ImmutableList.of("0", "1")); @@ -157,7 +168,12 @@ public void testOneLongColumn() @Test public void testOneFloatColumn() { - assertFilterMatches(EDF("dim2 == ''"), ImmutableList.of("0")); + if (NullHandling.replaceWithDefault()) { + assertFilterMatches(EDF("dim2 == ''"), ImmutableList.of("0")); + } else { + // A float does not match empty string + assertFilterMatches(EDF("dim2 == ''"), ImmutableList.of()); + } assertFilterMatches(EDF("dim2 == '1'"), ImmutableList.of("1")); assertFilterMatches(EDF("dim2 == 2"), ImmutableList.of("2")); assertFilterMatches(EDF("dim2 < '2'"), ImmutableList.of("0", "1")); @@ -179,11 +195,19 @@ public void testCompareColumns() // String vs string assertFilterMatches(EDF("dim0 == dim3"), ImmutableList.of("2", "5", "8")); - // String vs long - assertFilterMatches(EDF("dim1 == dim3"), ImmutableList.of("0", "2", "5", "8")); + if (NullHandling.replaceWithDefault()) { + // String vs long + assertFilterMatches(EDF("dim1 == dim3"), ImmutableList.of("0", "2", "5", "8")); + + // String vs float + assertFilterMatches(EDF("dim2 == dim3"), ImmutableList.of("0", "2", "5", "8")); + } else { + // String vs long + assertFilterMatches(EDF("dim1 == dim3"), ImmutableList.of("2", "5", "8")); - // String vs float - assertFilterMatches(EDF("dim2 == dim3"), ImmutableList.of("0", "2", "5", "8")); + // String vs float + assertFilterMatches(EDF("dim2 == dim3"), ImmutableList.of("2", "5", "8")); + } // String vs. multi-value string // Expressions currently treat multi-valued arrays as nulls. diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index 19ba6c565eba..10c075333b2f 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -183,7 +183,11 @@ public void testLongSelectorUsingStringFunction() } CURRENT_ROW.set(ROW3); - Assert.assertEquals(0L, selector.getLong()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0L, selector.getLong()); + } else { + Assert.assertTrue(selector.isNull()); + } } @Test diff --git a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java index 5dc3135ed965..be1bf5464bd7 100644 --- a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java @@ -20,6 +20,7 @@ package io.druid.query.expression; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.Expr; import io.druid.math.expr.Parser; @@ -84,8 +85,8 @@ public void testRegexpExtract() public void testTimestampCeil() { assertExpr("timestamp_ceil(t, 'P1M')", DateTimes.of("2000-03-01").getMillis()); - assertExpr("timestamp_ceil(t, 'P1D','','America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis()); - assertExpr("timestamp_ceil(t, 'P1D','',CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D',null,'America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D',null,CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis()); assertExpr("timestamp_ceil(t, 'P1D','1970-01-01T01','Etc/UTC')", DateTimes.of("2000-02-04T01").getMillis()); } @@ -93,8 +94,8 @@ public void testTimestampCeil() public void testTimestampFloor() { assertExpr("timestamp_floor(t, 'P1M')", DateTimes.of("2000-02-01").getMillis()); - assertExpr("timestamp_floor(t, 'P1D','','America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis()); - assertExpr("timestamp_floor(t, 'P1D','',CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D',null,'America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D',null,CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis()); assertExpr("timestamp_floor(t, 'P1D','1970-01-01T01','Etc/UTC')", DateTimes.of("2000-02-03T01").getMillis()); } @@ -122,12 +123,12 @@ public void testTimestampParse() assertExpr("timestamp_parse(tstr)", DateTimes.of("2000-02-03T04:05:06").getMillis()); assertExpr("timestamp_parse(tstr_sql)", DateTimes.of("2000-02-03T04:05:06").getMillis()); assertExpr( - "timestamp_parse(tstr_sql,'','America/Los_Angeles')", + "timestamp_parse(tstr_sql,null,'America/Los_Angeles')", DateTimes.of("2000-02-03T04:05:06-08:00").getMillis() ); assertExpr("timestamp_parse('2000-02-03')", DateTimes.of("2000-02-03").getMillis()); assertExpr("timestamp_parse('2000-02')", DateTimes.of("2000-02-01").getMillis()); - assertExpr("timestamp_parse('')", null); + assertExpr("timestamp_parse(null)", null); assertExpr("timestamp_parse('z2000')", null); assertExpr("timestamp_parse(tstr_sql,'yyyy-MM-dd HH:mm:ss')", DateTimes.of("2000-02-03T04:05:06").getMillis()); assertExpr("timestamp_parse('02/03/2000','MM/dd/yyyy')", DateTimes.of("2000-02-03").getMillis()); @@ -148,36 +149,39 @@ public void testTimestampFormat() @Test public void testTrim() { - assertExpr("trim('')", null); + String emptyString = NullHandling.replaceWithDefault() ? null : ""; + assertExpr("trim('')", emptyString); assertExpr("trim(concat(' ',x,' '))", "foo"); assertExpr("trim(spacey)", "hey there"); assertExpr("trim(spacey, '')", " hey there "); assertExpr("trim(spacey, 'he ')", "y ther"); - assertExpr("trim(spacey, spacey)", null); + assertExpr("trim(spacey, spacey)", emptyString); assertExpr("trim(spacey, substring(spacey, 0, 4))", "y ther"); } @Test public void testLTrim() { - assertExpr("ltrim('')", null); + String emptyString = NullHandling.replaceWithDefault() ? null : ""; + assertExpr("ltrim('')", emptyString); assertExpr("ltrim(concat(' ',x,' '))", "foo "); assertExpr("ltrim(spacey)", "hey there "); assertExpr("ltrim(spacey, '')", " hey there "); assertExpr("ltrim(spacey, 'he ')", "y there "); - assertExpr("ltrim(spacey, spacey)", null); + assertExpr("ltrim(spacey, spacey)", emptyString); assertExpr("ltrim(spacey, substring(spacey, 0, 4))", "y there "); } @Test public void testRTrim() { - assertExpr("rtrim('')", null); + String emptyString = NullHandling.replaceWithDefault() ? null : ""; + assertExpr("rtrim('')", emptyString); assertExpr("rtrim(concat(' ',x,' '))", " foo"); assertExpr("rtrim(spacey)", " hey there"); assertExpr("rtrim(spacey, '')", " hey there "); assertExpr("rtrim(spacey, 'he ')", " hey ther"); - assertExpr("rtrim(spacey, spacey)", null); + assertExpr("rtrim(spacey, spacey)", emptyString); assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther"); } From a5b6f9bea97bec9a2547b3019e72fa460b11509d Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 1 Mar 2018 17:06:03 +0530 Subject: [PATCH 08/38] fix tests remove unused import Uniformly set Calcite systemProperties in all tests formatting fix fix checkstyle extract CalciteSetSysteProperties at a common place --- .../druid/sql/avatica/DruidStatementTest.java | 16 +++++++- .../druid/sql/calcite/CalciteQueryTest.java | 13 ++++++- .../sql/calcite/http/SqlResourceTest.java | 2 +- .../druid/sql/calcite/util/CalciteTests.java | 37 +++++++++++-------- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java index a7b423515c0c..e2c9a218e550 100644 --- a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java @@ -144,7 +144,13 @@ public void testSelectAllInFirstFrame() throws Exception true, Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, - new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", NullHandling.defaultStringValue(), 2.0f}, + new Object[]{ + DateTimes.of("2000-01-02").getMillis(), + 1L, + "10.1", + NullHandling.defaultStringValue(), + 2.0f + }, new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, @@ -171,7 +177,13 @@ public void testSelectSplitOverTwoFrames() throws Exception false, Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, - new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", NullHandling.defaultStringValue(), 2.0f} + new Object[]{ + DateTimes.of("2000-01-02").getMillis(), + 1L, + "10.1", + NullHandling.defaultStringValue(), + 2.0f + } ) ), frame diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index c1d50121acfb..0d06392806f9 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -127,6 +127,7 @@ public class CalciteQueryTest extends CalciteTestBase { + private static final Logger log = new Logger(CalciteQueryTest.class); private static final PlannerConfig PLANNER_CONFIG_DEFAULT = new PlannerConfig(); @@ -523,7 +524,15 @@ public void testSelectStarOnForbiddenTable() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, NullHandling.defaultDoubleValue(), HLLCV1.class.getName()} + new Object[]{ + T("2000-01-01"), + 1L, + "forbidden", + "abcd", + 9999.0f, + NullHandling.defaultDoubleValue(), + HLLCV1.class.getName() + } ) ); } @@ -4197,7 +4206,7 @@ public void testTopNFilterJoin() throws Exception { DimFilter filter = NullHandling.replaceWithDefault() ? IN("dim2", Arrays.asList(null, "a"), null) - : SELECTOR("dim2", "a", null); + : SELECTOR("dim2", "a", null); // Filters on top N values of some dimension by using an inner join. testQuery( "SELECT t1.dim1, SUM(t1.cnt)\n" diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index f68714804fa6..9516bd1ce584 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -24,13 +24,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; import io.druid.math.expr.ExprMacroTable; import io.druid.query.QueryInterruptedException; import io.druid.query.ResourceLimitExceededException; -import io.druid.common.config.NullHandling; import io.druid.server.security.AllowAllAuthenticator; import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthTestUtils; diff --git a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java index bfff546e11d0..5261f62bfac6 100644 --- a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java @@ -32,8 +32,6 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; -import io.druid.java.util.emitter.core.NoopEmitter; -import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.collections.StupidPool; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; @@ -43,8 +41,9 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.guice.ExpressionModule; import io.druid.guice.annotations.Json; +import io.druid.java.util.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.math.expr.ExprMacroTable; -import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.DefaultGenericQueryMetricsFactory; import io.druid.query.DefaultQueryRunnerFactoryConglomerate; import io.druid.query.DruidProcessingConfig; @@ -92,12 +91,12 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndexSchema; +import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.server.QueryLifecycleFactory; import io.druid.server.log.NoopRequestLogger; import io.druid.server.security.Access; import io.druid.server.security.Action; import io.druid.server.security.AllowAllAuthenticator; -import io.druid.server.security.NoopEscalator; import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.Authenticator; @@ -105,6 +104,7 @@ import io.druid.server.security.Authorizer; import io.druid.server.security.AuthorizerMapper; import io.druid.server.security.Escalator; +import io.druid.server.security.NoopEscalator; import io.druid.server.security.Resource; import io.druid.server.security.ResourceType; import io.druid.sql.calcite.expression.SqlOperatorConversion; @@ -137,7 +137,8 @@ public class CalciteTests public static final String FORBIDDEN_DATASOURCE = "forbiddenDatasource"; public static final String TEST_SUPERUSER_NAME = "testSuperuser"; - public static final AuthorizerMapper TEST_AUTHORIZER_MAPPER = new AuthorizerMapper(null) { + public static final AuthorizerMapper TEST_AUTHORIZER_MAPPER = new AuthorizerMapper(null) + { @Override public Authorizer getAuthorizer(String name) { @@ -162,11 +163,13 @@ public Access authorize( } }; public static final AuthenticatorMapper TEST_AUTHENTICATOR_MAPPER; + static { final Map defaultMap = Maps.newHashMap(); defaultMap.put( AuthConfig.ALLOW_ALL_NAME, - new AllowAllAuthenticator() { + new AllowAllAuthenticator() + { @Override public AuthenticationResult authenticateJDBCContext(Map context) { @@ -176,9 +179,12 @@ public AuthenticationResult authenticateJDBCContext(Map context) ); TEST_AUTHENTICATOR_MAPPER = new AuthenticatorMapper(defaultMap); } + public static final Escalator TEST_AUTHENTICATOR_ESCALATOR; + static { - TEST_AUTHENTICATOR_ESCALATOR = new NoopEscalator() { + TEST_AUTHENTICATOR_ESCALATOR = new NoopEscalator() + { @Override public AuthenticationResult createEscalatedAuthenticationResult() @@ -215,15 +221,14 @@ public void configure(final Binder binder) // This Module is just to get a LookupReferencesManager with a usable "lookyloo" lookup. - binder.bind(LookupReferencesManager.class) - .toInstance( - LookupEnabledTestExprMacroTable.createTestLookupReferencesManager( - ImmutableMap.of( - "a", "xa", - "abc", "xabc" - ) - ) - ); + binder.bind(LookupReferencesManager.class).toInstance( + LookupEnabledTestExprMacroTable.createTestLookupReferencesManager( + ImmutableMap.of( + "a", "xa", + "abc", "xabc" + ) + ) + ); } } From 97f6114190ed8f1de5412160a6dea20b2af08f76 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 2 Mar 2018 22:56:55 +0530 Subject: [PATCH 09/38] revert unintentional change --- .../query/aggregation/last/DoubleLastAggregatorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 90a3e800799f..2eec8764678c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -151,7 +151,7 @@ public void aggregate(ByteBuffer buf, int position) long lastTime = buf.getLong(position); if (pair.lhs >= lastTime) { buf.putLong(position, pair.lhs); - buf.putDouble(position + Double.BYTES, pair.rhs); + buf.putDouble(position + Long.BYTES, pair.rhs); } } From 351aefb1684e02037b720a3f0067648cfc6eeff2 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 16 Mar 2018 03:31:00 +0530 Subject: [PATCH 10/38] review comments --- .../benchmark/FilterPartitionBenchmark.java | 4 +- codestyle/checkstyle.xml | 10 ++++ .../main/java/io/druid/math/expr/Expr.java | 4 +- .../java/io/druid/math/expr/ExprEval.java | 27 +++++----- .../java/io/druid/math/expr/Function.java | 2 +- .../sql/QuantileSqlAggregatorTest.java | 12 +++-- .../namespace/UriExtractionNamespace.java | 3 ++ .../cache/JdbcExtractionNamespaceTest.java | 8 +-- .../io/druid/server/lookup/LoadingLookup.java | 9 ++-- .../io/druid/server/lookup/PollingLookup.java | 6 ++- .../server/lookup/jdbc/JdbcDataFetcher.java | 4 +- .../server/lookup/PollingLookupTest.java | 6 +-- .../main/java/io/druid/indexer/JobHelper.java | 3 ++ .../io/druid/indexer/path/StaticPathSpec.java | 3 ++ .../common/task/RealtimeIndexTaskTest.java | 4 +- .../java/util/common/parsers/ParserUtils.java | 4 ++ .../util/http/client/NettyHttpClient.java | 4 +- .../io/druid/query/DefaultQueryMetrics.java | 1 + .../query/aggregation/AggregatorFactory.java | 2 +- .../query/aggregation/AggregatorUtil.java | 12 ++--- .../DoubleMaxAggregatorFactory.java | 4 +- .../DoubleMinAggregatorFactory.java | 4 +- .../DoubleSumAggregatorFactory.java | 4 +- .../FloatMaxAggregatorFactory.java | 4 +- .../FloatMinAggregatorFactory.java | 4 +- .../FloatSumAggregatorFactory.java | 4 +- .../aggregation/LongMaxAggregatorFactory.java | 4 +- .../aggregation/LongSumAggregatorFactory.java | 4 +- .../NullableAggregateCombiner.java | 10 ++-- .../query/aggregation/NullableAggregator.java | 2 +- .../NullableAggregatorFactory.java | 50 ++++++++++++++++--- .../aggregation/NullableBufferAggregator.java | 6 +-- .../first/FloatFirstAggregatorFactory.java | 4 +- .../first/LongFirstAggregatorFactory.java | 4 +- .../last/DoubleLastAggregatorFactory.java | 4 +- .../last/FloatLastAggregatorFactory.java | 4 +- .../last/LongLastAggregatorFactory.java | 4 +- .../ExpressionColumnValueSelector.java | 2 +- .../segment/virtual/ExpressionSelectors.java | 2 +- ...tCachingExpressionColumnValueSelector.java | 2 +- ...tCachingExpressionColumnValueSelector.java | 2 +- ...roupByLimitPushDownMultiNodeMergeTest.java | 3 +- .../query/lookup/LookupExtractionFnTest.java | 16 +++--- .../query/dimension/LookupDimensionSpec.java | 7 ++- .../query/expression/LookupExprMacro.java | 4 +- .../io/druid/query/lookup/LookupModule.java | 4 ++ .../java/io/druid/server/QueryLifecycle.java | 8 ++- .../druid/server/emitter/EmitterModule.java | 3 ++ .../announcer/ListeningAnnouncerConfig.java | 3 ++ .../dimension/LookupDimensionSpecTest.java | 2 + .../sql/calcite/planner/DruidRexExecutor.java | 4 +- 51 files changed, 191 insertions(+), 119 deletions(-) diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java index bd7e8415495e..a3b9439b3ed6 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java @@ -22,11 +22,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.io.Files; import io.druid.benchmark.datagen.BenchmarkDataGenerator; import io.druid.benchmark.datagen.BenchmarkSchemaInfo; import io.druid.benchmark.datagen.BenchmarkSchemas; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogHash; import io.druid.jackson.DefaultObjectMapper; @@ -610,7 +610,7 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { - final String valueOrNull = Strings.emptyToNull(value); + final String valueOrNull = NullHandling.emptyToNullIfNeeded(value); final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index 9b8a9ed3eab7..f2695f3fcbf6 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -170,5 +170,15 @@ + + + + + + + + + + diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 690146788eae..ae55e7d4dcb2 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -372,12 +372,12 @@ public ExprEval eval(ObjectBinding bindings) if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) { return evalString(leftVal.asString(), rightVal.asString()); } else if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) { - if (NullHandling.sqlCompatible() && (leftVal.isNull() || rightVal.isNull())) { + if (NullHandling.sqlCompatible() && (leftVal.isNumericNull() || rightVal.isNumericNull())) { return ExprEval.of(null); } return ExprEval.of(evalLong(leftVal.asLong(), rightVal.asLong())); } else { - if (NullHandling.sqlCompatible() && (leftVal.isNull() || rightVal.isNull())) { + if (NullHandling.sqlCompatible() && (leftVal.isNumericNull() || rightVal.isNumericNull())) { return ExprEval.of(null); } return ExprEval.of(evalDouble(leftVal.asDouble(), rightVal.asDouble())); diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 3906fc945aeb..195b74dc2747 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -30,7 +30,7 @@ */ public abstract class ExprEval { - public static ExprEval ofLong(Number longValue) + public static ExprEval ofLong(@Nullable Number longValue) { return new LongExprEval(longValue); } @@ -40,7 +40,7 @@ public static ExprEval of(long longValue) return new LongExprEval(longValue); } - public static ExprEval ofDouble(Number doubleValue) + public static ExprEval ofDouble(@Nullable Number doubleValue) { return new DoubleExprEval(doubleValue); } @@ -69,7 +69,7 @@ public static ExprEval of(boolean value, ExprType type) } } - public static ExprEval bestEffortOf(Object val) + public static ExprEval bestEffortOf(@Nullable Object val) { if (val instanceof ExprEval) { return (ExprEval) val; @@ -83,7 +83,7 @@ public static ExprEval bestEffortOf(Object val) return new StringExprEval(val == null ? null : String.valueOf(val)); } - final T value; + @Nullable final T value; private ExprEval(T value) { @@ -100,10 +100,7 @@ public Object value() /** * returns true if numeric primitive value for this ExprEval is null, otherwise false. */ - public boolean isNull() - { - return value == null; - } + public abstract boolean isNumericNull(); public abstract int asInt(); @@ -126,7 +123,7 @@ public String asString() private abstract static class NumericExprEval extends ExprEval { - private NumericExprEval(Number value) + private NumericExprEval(@Nullable Number value) { super(value == null ? NullHandling.defaultDoubleValue() : value); } @@ -148,11 +145,17 @@ public final double asDouble() { return value.doubleValue(); } + + @Override + public boolean isNumericNull() + { + return value == null; + } } private static class DoubleExprEval extends NumericExprEval { - private DoubleExprEval(Number value) + private DoubleExprEval(@Nullable Number value) { super(value); } @@ -192,7 +195,7 @@ public Expr toExpr() private static class LongExprEval extends NumericExprEval { - private LongExprEval(Number value) + private LongExprEval(@Nullable Number value) { super(value); } @@ -292,7 +295,7 @@ private Number asNumber() } @Override - public boolean isNull() + public boolean isNumericNull() { return asNumber() == null; } diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 672dae0315d3..4116030d6288 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -74,7 +74,7 @@ abstract class SingleParamMath extends SingleParam @Override protected final ExprEval eval(ExprEval param) { - if (NullHandling.sqlCompatible() && param.isNull()) { + if (NullHandling.sqlCompatible() && param.isNumericNull()) { return ExprEval.of(null); } if (param.type() == ExprType.LONG) { diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 1ceab0e529ec..d18917675a88 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.granularity.Granularities; import io.druid.query.Druids; import io.druid.query.QueryDataSource; @@ -306,9 +307,14 @@ public void testQuantileOnInnerQuery() throws Exception // Verify results final List results = plannerResult.run().toList(); - final List expectedResults = ImmutableList.of( - new Object[]{7.0, 8.26386833190918} - ); + final List expectedResults = + NullHandling.replaceWithDefault() ? + ImmutableList.of( + new Object[]{7.0, 8.26386833190918} + ) : + ImmutableList.of( + new Object[]{5.25, 6.59091854095459} + ); Assert.assertEquals(expectedResults.size(), results.size()); for (int i = 0; i < expectedResults.size(); i++) { Assert.assertArrayEquals(expectedResults.get(i), results.get(i)); diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index ff0a93c4da2c..7033a6e68b8e 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -395,12 +395,15 @@ public TSVFlatDataParser( Preconditions.checkNotNull(columns, "`columns` list required").size() > 1, "Must specify more than one column to have a key value pair" ); + //CHECKSTYLE.OFF: Regexp + // String.emptytoNull usage here is irrelevant to null handling of the data. final DelimitedParser delegate = new DelimitedParser( Strings.emptyToNull(delimiter), Strings.emptyToNull(listDelimiter), hasHeaderRow, skipHeaderRows ); + //CHECKSTYLE.ON: Regexp Preconditions.checkArgument( !(Strings.isNullOrEmpty(keyColumn) ^ Strings.isNullOrEmpty(valueColumn)), "Must specify both `keyColumn` and `valueColumn` or neither `keyColumn` nor `valueColumn`" diff --git a/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java b/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java index 08cc4ee6ee20..1339f8cae4c7 100644 --- a/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java +++ b/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java @@ -19,13 +19,13 @@ package io.druid.server.lookup.namespace.cache; -import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.Closer; @@ -384,7 +384,7 @@ public void testMappingWithoutFilter() String key = e.getKey(); String[] val = e.getValue(); String field = val[0]; - Assert.assertEquals("non-null check", Strings.emptyToNull(field), Strings.emptyToNull(map.get(key))); + Assert.assertEquals("non-null check", NullHandling.emptyToNullIfNeeded(field), NullHandling.emptyToNullIfNeeded(map.get(key))); } Assert.assertEquals("null check", null, map.get("baz")); } @@ -415,9 +415,9 @@ public void testMappingWithFilter() String filterVal = val[1]; if (filterVal.equals("1")) { - Assert.assertEquals("non-null check", Strings.emptyToNull(field), Strings.emptyToNull(map.get(key))); + Assert.assertEquals("non-null check", NullHandling.emptyToNullIfNeeded(field), NullHandling.emptyToNullIfNeeded(map.get(key))); } else { - Assert.assertEquals("non-null check", null, Strings.emptyToNull(map.get(key))); + Assert.assertEquals("non-null check", null, NullHandling.emptyToNullIfNeeded(map.get(key))); } } } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java index 3534b322db3a..201b5418410c 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java @@ -21,12 +21,13 @@ import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.logger.Logger; import io.druid.query.lookup.LookupExtractor; import io.druid.server.lookup.cache.loading.LoadingCache; +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; import java.util.concurrent.Callable; @@ -63,6 +64,7 @@ public LoadingLookup( @Override + @Nullable public String apply(final String key) { if (key == null) { @@ -71,7 +73,7 @@ public String apply(final String key) final String presentVal; try { presentVal = loadingCache.get(key, new ApplyCallable(key)); - return Strings.emptyToNull(presentVal); + return NullHandling.emptyToNullIfNeeded(presentVal); } catch (ExecutionException e) { LOGGER.debug("value not found for key [%s]", key); @@ -132,8 +134,9 @@ public ApplyCallable(String key) @Override public String call() throws Exception { + // When SQL based null handling is disabled, // avoid returning null and return an empty string to cache it. - return Strings.nullToEmpty(dataFetcher.fetch(key)); + return NullHandling.nullToEmptyIfNeeded(dataFetcher.fetch(key)); } } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java index 0abc23b8e226..fb31d925aee0 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java @@ -20,11 +20,11 @@ package io.druid.server.lookup; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; @@ -33,6 +33,7 @@ import io.druid.server.lookup.cache.polling.PollingCache; import io.druid.server.lookup.cache.polling.PollingCacheFactory; +import javax.annotation.Nullable; import javax.validation.constraints.NotNull; import java.util.List; import java.util.concurrent.Executors; @@ -107,6 +108,7 @@ public void close() } @Override + @Nullable public String apply(@NotNull String key) { final CacheRefKeeper cacheRefKeeper = refOfCacheKeeper.get(); @@ -119,7 +121,7 @@ public String apply(@NotNull String key) // it must've been closed after swapping while I was getting it. Try again. return this.apply(key); } - return Strings.emptyToNull((String) cache.get(key)); + return NullHandling.emptyToNullIfNeeded((String) cache.get(key)); } finally { if (cacheRefKeeper != null && cache != null) { diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java index 6d0af4e821fb..e1ac561d4ff1 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java @@ -21,9 +21,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.metadata.MetadataStorageConnectorConfig; @@ -132,7 +132,7 @@ public List inTransaction(Handle handle, TransactionStatus status) throw if (pairs.isEmpty()) { return null; } - return Strings.nullToEmpty(pairs.get(0)); + return NullHandling.nullToEmptyIfNeeded(pairs.get(0)); } @Override diff --git a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java index dee07f60332e..4afaed01feae 100644 --- a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java +++ b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java @@ -21,11 +21,11 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.ISE; import io.druid.query.lookup.LookupExtractor; import io.druid.server.lookup.cache.polling.OffHeapPollingCache; @@ -191,7 +191,7 @@ public void testBulkApply() public String apply(String input) { //make sure to rewrite null strings as empty. - return Strings.nullToEmpty(input); + return NullHandling.nullToEmptyIfNeeded(input); } })); } @@ -208,7 +208,7 @@ private void assertMapLookup(Map map, LookupExtractor lookup) for (Map.Entry entry : map.entrySet()) { String key = entry.getKey(); String val = entry.getValue(); - Assert.assertEquals("non-null check", Strings.emptyToNull(val), lookup.apply(key)); + Assert.assertEquals("non-null check", NullHandling.emptyToNullIfNeeded(val), lookup.apply(key)); } } } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 16809561cbbc..80d0f1aaa506 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -302,8 +302,11 @@ public static void injectSystemProperties(Job job) public static void injectDruidProperties(Configuration configuration, List listOfAllowedPrefix) { + //CHECKSTYLE.OFF: Regexp + // String.nulltoEmpty usage here is irrelevant to null handling of the data. String mapJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.MAP_JAVA_OPTS)); String reduceJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.REDUCE_JAVA_OPTS)); + //CHECKSTYLE.ON: regexp for (String propName : System.getProperties().stringPropertyNames()) { for (String prefix : listOfAllowedPrefix) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java index 37519fd67e4b..74c4800aa56d 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java @@ -128,7 +128,10 @@ public static void addToMultipleInputs( private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass) { Configuration conf = job.getConfiguration(); + //CHECKSTYLE.OFF: Regexp + // String.nulltoempty usage here is irrelevant to null handling of the data. StringBuilder inputFormats = new StringBuilder(Strings.nullToEmpty(conf.get(MultipleInputs.DIR_FORMATS))); + //CHECKSTYLE.ON: Regexp String[] paths = Iterables.toArray(pathStrings, String.class); for (int i = 0; i < paths.length - 1; i++) { diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index 44cbda829f28..22a4891bec30 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -120,6 +120,7 @@ import io.druid.server.coordination.DataSegmentServerAnnouncer; import io.druid.server.coordination.ServerType; import io.druid.timeline.DataSegment; +import io.druid.utils.Runnables; import org.easymock.EasyMock; import org.hamcrest.CoreMatchers; import org.joda.time.DateTime; @@ -208,8 +209,7 @@ public InputRow nextRow() @Override public Runnable commit() { - return () -> { - }; + return Runnables.getNoopRunnable(); } @Override diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java index e3975ca15d44..6859c70c4786 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java @@ -58,6 +58,9 @@ public static Function getMultiValueFunction( ) { return (input) -> { + //CHECKSTYLE.OFF: Regexp + // TODO: java-util does not depen on druid-common.NullHandling is not accessible here. + // Fix null semantics for CSV and TSV parsers. if (input != null && input.contains(listDelimiter)) { return StreamSupport.stream(listSplitter.split(input).spliterator(), false) .map(Strings::emptyToNull) @@ -65,6 +68,7 @@ public static Function getMultiValueFunction( } else { return Strings.emptyToNull(input); } + //CHECKSTYLE.ON: Regexp }; } diff --git a/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java index 0d343a06f1ab..a02c6aecffe3 100644 --- a/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java +++ b/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java @@ -150,8 +150,10 @@ public ListenableFuture go( } else { channel = channelFuture.getChannel(); } - + //CHECKSTYLE.OFF: Regexp + // String.nullToEmpty usage here is irrelevant to null handling of the data. final String urlFile = Strings.nullToEmpty(url.getFile()); + //CHECKSTYLE.ON: Regexp final HttpRequest httpRequest = new DefaultHttpRequest( HttpVersion.HTTP_1_1, method, diff --git a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java index b86ee99bb7d5..0df370de62d8 100644 --- a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java @@ -119,6 +119,7 @@ public void duration(QueryType query) public void queryId(QueryType query) { //CHECKSTYLE.OFF: Regexp + // String.nullToEmpty usage here is irrelevant to null handling of the data. setDimension(DruidMetrics.ID, Strings.nullToEmpty(query.getId())); //CHECKSTYLE.ON: Regexp } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index cadde722e50b..ffe46400d868 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -36,7 +36,7 @@ * AggregatorFactory is a strategy (in the terms of Design Patterns) that represents column aggregation, e. g. min, * max, sum of metric columns, or cardinality of dimension columns (see {@link * io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory}). - * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * Implementations of {@link AggregatorFactory} which need to Support Nullable Aggregations are encouraged * to extend {@link io.druid.query.aggregation.NullableAggregatorFactory}. */ @ExtensionPoint diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 5ba40191218b..35bf50793756 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -154,7 +154,7 @@ class ExpressionFloatColumnSelector implements FloatColumnSelector public float getFloat() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval.isNull() ? nullValue : (float) exprEval.asDouble(); + return exprEval.isNumericNull() ? nullValue : (float) exprEval.asDouble(); } @Override @@ -167,7 +167,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public boolean isNull() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval == null || exprEval.isNull(); + return exprEval == null || exprEval.isNumericNull(); } } return new ExpressionFloatColumnSelector(); @@ -195,7 +195,7 @@ class ExpressionLongColumnSelector implements LongColumnSelector public long getLong() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval.isNull() ? nullValue : exprEval.asLong(); + return exprEval.isNumericNull() ? nullValue : exprEval.asLong(); } @Override @@ -208,7 +208,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public boolean isNull() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval == null || exprEval.isNull(); + return exprEval == null || exprEval.isNumericNull(); } } return new ExpressionLongColumnSelector(); @@ -236,7 +236,7 @@ class ExpressionDoubleColumnSelector implements DoubleColumnSelector public double getDouble() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval.isNull() ? nullValue : exprEval.asDouble(); + return exprEval.isNumericNull() ? nullValue : exprEval.asDouble(); } @Override @@ -249,7 +249,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) public boolean isNull() { final ExprEval exprEval = baseSelector.getObject(); - return exprEval == null || exprEval.isNull(); + return exprEval == null || exprEval.isNumericNull(); } } return new ExpressionDoubleColumnSelector(); diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 49cda751c7a1..964bef68c613 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -69,9 +69,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new DoubleMaxBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 46ede137418a..96b96baaac47 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -68,9 +68,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new DoubleMinBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index bc5735c825e1..b3ca542ff282 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -69,9 +69,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new DoubleSumBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index 8ae3c647bf35..bd02b2a7a08e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -68,9 +68,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new FloatMaxBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index 8b3b3334e25b..be11112faa92 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -68,9 +68,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new FloatMinBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index a687ea653181..a34f109ee1aa 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -68,9 +68,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new FloatSumBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index c78365cdaa5f..f6d8fedce9d7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -84,9 +84,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new LongMaxBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 46f1b8b29d8b..09f5c77d7572 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -84,9 +84,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new LongSumBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index e33b7b458a4f..23eb789ad2c6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -26,7 +26,7 @@ /** * The result of a NullableAggregateCombiner will be null if all the values to be combined are null values or no values are combined at all. * If any of the value is non-null, the result would be the value of the delegate combiner. - * Note that the delegate combiner is not required to perform check for isNull on the columnValueSelector as only non-null values + * Note that the delegate combiner is not required to perform check for {@link NullableAggregateCombiner#isNull} on the columnValueSelector as only non-null values * will be passed to the delegate combiner. */ public class NullableAggregateCombiner implements AggregateCombiner @@ -54,11 +54,9 @@ public void reset(ColumnValueSelector selector) @Override public void fold(ColumnValueSelector selector) { - boolean isNotNull = !selector.isNull(); - if (isNotNull) { - if (isNullResult) { - isNullResult = false; - } + if (isNullResult) { + reset(selector); + } else { delegate.fold(selector); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 1eac57edc34b..dea9cdf4732e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -26,7 +26,7 @@ /** * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. - * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * Note that the delegate aggregator is not required to perform check for {@link NullableAggregator#isNull} on the columnValueSelector as only non-null values * will be passed to the delegate aggregator. */ public class NullableAggregator implements Aggregator diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 1852037296ae..4cda2951d21f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -21,14 +21,16 @@ import io.druid.common.config.NullHandling; +import io.druid.guice.annotations.ExtensionPoint; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; /** - * abstract class with functionality to wrap aggregator/bufferAggregator/combiner to make them Nullable. - * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * abstract class with functionality to wrap {@link Aggregator}, {@link BufferAggregator} and {@link AggregateCombiner} to make them Nullable. + * Implementations of {@link AggregatorFactory} which need to Support Nullable Aggregations are encouraged * to extend this class. */ +@ExtensionPoint public abstract class NullableAggregatorFactory extends AggregatorFactory { @Override @@ -47,11 +49,6 @@ public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFact return NullHandling.replaceWithDefault() ? aggregator : new NullableBufferAggregator(aggregator, selector); } - protected abstract BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, - ColumnValueSelector selector - ); - @Override public final AggregateCombiner makeAggregateCombiner() { @@ -67,12 +64,51 @@ public final int getMaxIntermediateSize() // ---- ABSTRACT METHODS BELOW ------ + /** + * Creates a {@link ColumnValueSelector} for the aggregated column. + * + * @see ColumnValueSelector + */ protected abstract ColumnValueSelector selector(ColumnSelectorFactory metricFactory); + /** + * Creates an {@link Aggregator} to aggregate values from several rows, by using the provided selector. + * @param metricFactory metricFactory + * @param selector {@link ColumnValueSelector} for the column to aggregate. + * + * @see Aggregator + */ protected abstract Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); + /** + * Creates an {@link BufferAggregator} to aggregate values from several rows into a ByteBuffer. + * @param metricFactory metricFactory + * @param selector {@link ColumnValueSelector} for the column to aggregate. + * + * @see BufferAggregator + */ + protected abstract BufferAggregator factorizeBuffered( + ColumnSelectorFactory metricFactory, + ColumnValueSelector selector + ); + + /** + * Creates an {@link AggregateCombiner} to fold rollup aggregation results from serveral "rows" of different indexes during + * index merging. AggregateCombiner implements the same logic as {@link #combine}, with the difference that it uses + * {@link io.druid.segment.ColumnValueSelector} and it's subinterfaces to get inputs and implements {@code + * ColumnValueSelector} to provide output. + * + * @see AggregateCombiner + * @see io.druid.segment.IndexMerger + */ + @SuppressWarnings("unused") // Going to be used when https://github.com/druid-io/druid/projects/2 is complete protected abstract AggregateCombiner makeAggregateCombiner2(); + /** + * Returns the maximum size that this aggregator will require in bytes for intermediate storage of results. + * + * @return the maximum number of bytes that an aggregator of this type will require for intermediate result storage. + */ protected abstract int getMaxIntermediateSize2(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index cd27bc6039d0..6447f4d24671 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -27,13 +27,13 @@ /** * The result of a NullableBufferAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. - * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * Note that the delegate aggregator is not required to perform check for {@link NullableBufferAggregator#isNull(ByteBuffer, int)} on the columnValueSelector as only non-null values * will be passed to the delegate aggregator. */ public class NullableBufferAggregator implements BufferAggregator { - private static final byte IS_NULL_BYTE = (byte) 1; - private static final byte IS_NOT_NULL_BYTE = (byte) 0; + private static final byte IS_NULL_BYTE = (byte) 0; + private static final byte IS_NOT_NULL_BYTE = (byte) 1; private final BufferAggregator delegate; private final BaseNullableColumnValueSelector selector; diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 9ec538b306d9..ed67d345f7ad 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -90,9 +90,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new FloatFirstBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index e4d2e958a94d..9554662efa63 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -83,9 +83,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new LongFirstBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 2eec8764678c..8edc224d544c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -82,9 +82,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new DoubleLastBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 34d40eecc73e..c83f491dd9ab 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -80,9 +80,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new FloatLastBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 3cc6455b7591..d927fdfc036f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -79,9 +79,7 @@ protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueS } @Override - protected BufferAggregator factorizeBuffered( - ColumnSelectorFactory metricFactory, ColumnValueSelector selector - ) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) { return new LongLastBufferAggregator( metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java index 18c64d1bfcdd..297ecce3714a 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java @@ -82,6 +82,6 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public boolean isNull() { - return getObject().isNull(); + return getObject().isNumericNull(); } } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 3aae3b797d7c..fe67f65adad2 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -161,7 +161,7 @@ public static ColumnValueSelector makeExprEvalSelector( if (bindings.equals(ExprUtils.nilBindings())) { // Optimization for constant expressions. final ExprEval eval = expression.eval(bindings); - if (NullHandling.sqlCompatible() && eval.isNull()) { + if (NullHandling.sqlCompatible() && eval.isNumericNull()) { return NilColumnValueSelector.instance(); } return new ConstantColumnValueSelector<>( diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java index 2a0be1ccf767..e2d504bd70b8 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java @@ -150,6 +150,6 @@ private ExprEval eval(final long value) @Override public boolean isNull() { - return getObject().isNull(); + return getObject().isNumericNull(); } } diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java index 2b2db902d9ea..ae5fb3467d70 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java @@ -139,7 +139,7 @@ private ExprEval eval() @Override public boolean isNull() { - return eval().isNull(); + return eval().isNumericNull(); } public static class LruEvalCache diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index ed51341d0543..ad8f783b7037 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -93,6 +93,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -561,7 +562,7 @@ public Sequence run(QueryPlus queryPlus, Map responseC "d2", 13L, "a0", 2L ); - + System.out.println(results); Assert.assertEquals(4, results.size()); Assert.assertEquals(expectedRow0, results.get(0)); Assert.assertEquals(expectedRow1, results.get(1)); diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java index aea8bf626d6d..7fa56973971c 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java @@ -20,13 +20,13 @@ package io.druid.query.lookup; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.IAE; import io.druid.query.extraction.MapLookupExtractor; @@ -66,7 +66,7 @@ public static Iterable constructorFeeder() public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, Optional injective) { - this.replaceMissing = Strings.emptyToNull(replaceMissing); + this.replaceMissing = NullHandling.emptyToNullIfNeeded(replaceMissing); this.retainMissing = retainMissing; this.injective = injective.orElse(null); } @@ -74,7 +74,7 @@ public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, Opti @Test public void testEqualsAndHash() { - if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { + if (retainMissing && !NullHandling.isNullOrEquivalent(replaceMissing)) { // skip return; } @@ -111,7 +111,7 @@ public void testEqualsAndHash() @Test public void testSimpleSerDe() throws IOException { - if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { + if (retainMissing && !NullHandling.isNullOrEquivalent(replaceMissing)) { // skip return; } @@ -146,12 +146,12 @@ public void testSimpleSerDe() throws IOException @Test(expected = IllegalArgumentException.class) public void testIllegalArgs() { - if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { + if (retainMissing && !NullHandling.isNullOrEquivalent(replaceMissing)) { @SuppressWarnings("unused") // expected exception final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, - Strings.emptyToNull(replaceMissing), + NullHandling.emptyToNullIfNeeded(replaceMissing), injective, false ); @@ -163,7 +163,7 @@ public void testIllegalArgs() @Test public void testCacheKey() { - if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { + if (retainMissing && !NullHandling.isNullOrEquivalent(replaceMissing)) { // skip return; } @@ -178,7 +178,7 @@ public void testCacheKey() false ); - if (Strings.isNullOrEmpty(replaceMissing) || retainMissing) { + if (NullHandling.isNullOrEquivalent(replaceMissing) || retainMissing) { Assert.assertFalse( Arrays.equals( lookupExtractionFn.getCacheKey(), diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index 97448a55cb10..cf1e9216cff9 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.DimFilterUtils; @@ -77,7 +78,7 @@ public LookupDimensionSpec( { this.retainMissingValue = retainMissingValue; this.optimize = optimize == null ? true : optimize; - this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandling.emptyToNullIfNeeded(replaceMissingValueWith); this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be Null"); this.outputName = Preconditions.checkNotNull(outputName, "outputName can not be Null"); this.lookupReferencesManager = lookupReferencesManager; @@ -166,12 +167,16 @@ public boolean mustDecorate() @Override public byte[] getCacheKey() { + byte[] dimensionBytes = StringUtils.toUtf8(dimension); byte[] dimExtractionFnBytes = Strings.isNullOrEmpty(name) ? getLookup().getCacheKey() : StringUtils.toUtf8(name); byte[] outputNameBytes = StringUtils.toUtf8(outputName); + //CHECKSTYLE.OFF: Regexp + // String.nullToEmpty usage here is irrelevant to null handling of the data. byte[] replaceWithBytes = StringUtils.toUtf8(Strings.nullToEmpty(replaceMissingValueWith)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(6 diff --git a/server/src/main/java/io/druid/query/expression/LookupExprMacro.java b/server/src/main/java/io/druid/query/expression/LookupExprMacro.java index 77dc88c184bd..e24ad3ff8c71 100644 --- a/server/src/main/java/io/druid/query/expression/LookupExprMacro.java +++ b/server/src/main/java/io/druid/query/expression/LookupExprMacro.java @@ -19,8 +19,8 @@ package io.druid.query.expression; -import com.google.common.base.Strings; import com.google.inject.Inject; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -77,7 +77,7 @@ class LookupExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - return ExprEval.of(extractionFn.apply(Strings.emptyToNull(arg.eval(bindings).asString()))); + return ExprEval.of(extractionFn.apply(NullHandling.emptyToNullIfNeeded(arg.eval(bindings).asString()))); } @Override diff --git a/server/src/main/java/io/druid/query/lookup/LookupModule.java b/server/src/main/java/io/druid/query/lookup/LookupModule.java index a1d387cf7225..72c0bc861a70 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupModule.java +++ b/server/src/main/java/io/druid/query/lookup/LookupModule.java @@ -260,11 +260,15 @@ public String getLookupTier() "Cannot specify both `lookupTier` and `lookupTierIsDatasource`" ); final String lookupTier = lookupTierIsDatasource ? dataSourceTaskIdHolder.getDataSource() : this.lookupTier; + + //CHECKSTYLE.OFF: Regexp + // String.emptytoNull usage here is irrelevant to null handling of the data. return Preconditions.checkNotNull( lookupTier == null ? DEFAULT_TIER : Strings.emptyToNull(lookupTier), "Cannot have empty lookup tier from %s", lookupTierIsDatasource ? "bound value" : LookupModule.PROPERTY_BASE ); + //CHECKSTYLE.ON: Regexp } public String getLookupKey() diff --git a/server/src/main/java/io/druid/server/QueryLifecycle.java b/server/src/main/java/io/druid/server/QueryLifecycle.java index 435e376dfd3d..faf62ea8617f 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycle.java +++ b/server/src/main/java/io/druid/server/QueryLifecycle.java @@ -288,12 +288,16 @@ public void emitLogsAndMetrics( try { final long queryTimeNs = System.nanoTime() - startNs; + + //CHECKSTYLE.OFF: Regexp + // String.nullToEmpty usage here is irrelevant to null handling of the data. QueryMetrics queryMetrics = DruidMetrics.makeRequestMetrics( queryMetricsFactory, toolChest, baseQuery, Strings.nullToEmpty(remoteAddress) ); + //CHECKSTYLE.ON: Regexp queryMetrics.success(success); queryMetrics.reportQueryTime(queryTimeNs); @@ -326,7 +330,8 @@ public void emitLogsAndMetrics( statsMap.put("reason", e.toString()); } } - + //CHECKSTYLE.OFF: Regexp + // String.nullToEmpty usage here is irrelevant to null handling of the data. requestLogger.log( new RequestLogLine( DateTimes.utc(startMs), @@ -335,6 +340,7 @@ public void emitLogsAndMetrics( new QueryStats(statsMap) ) ); + //CHECKSTYLE.ON: Regexp } catch (Exception ex) { log.error(ex, "Unable to log query [%s]!", baseQuery); diff --git a/server/src/main/java/io/druid/server/emitter/EmitterModule.java b/server/src/main/java/io/druid/server/emitter/EmitterModule.java index 7df0137b59c3..dfed77b1b2f0 100644 --- a/server/src/main/java/io/druid/server/emitter/EmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/EmitterModule.java @@ -86,9 +86,12 @@ public void configure(Binder binder) ExtraServiceDimensions.class ); String version = getClass().getPackage().getImplementationVersion(); + //CHECKSTYLE.OFF: Regexp + // String.nullToempty usage here is irrelevant to null handling of the data. extraServiceDimensions .addBinding("version") .toInstance(Strings.nullToEmpty(version)); // Version is null during `mvn test`. + //CHECKSTYLE.ON: Regexp } @Provides diff --git a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java index 35288b6bc3f9..f3db4d57c900 100644 --- a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java +++ b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java @@ -91,10 +91,13 @@ public String toString() */ public String getAnnouncementPath(String listenerName) { + //CHECKSTYLE.OFF: Regexp + // String.nullToempty usage here is irrelevant to null handling of the data. return ZKPaths.makePath( getListenersPath(), Preconditions.checkNotNull( Strings.emptyToNull(listenerName), "Listener name cannot be null" ) ); + //CHECKSTYPE.OFF: Regexp } } diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index 7e1ef423f959..ddd2108f4b25 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -160,7 +160,9 @@ public Object[] parametersForTestApply() public void testApply(DimensionSpec dimensionSpec, Map map) { for (Map.Entry entry : map.entrySet()) { + //CHECKSTYLE.OFF: Regexp Assert.assertEquals(Strings.emptyToNull(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); + //CHECKSTYLE.ON: Regexp } } diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/DruidRexExecutor.java b/sql/src/main/java/io/druid/sql/calcite/planner/DruidRexExecutor.java index 1e187f78742b..093befbbf620 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/DruidRexExecutor.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/DruidRexExecutor.java @@ -84,7 +84,7 @@ public void reduce( if (sqlTypeName == SqlTypeName.BOOLEAN) { literal = rexBuilder.makeLiteral(exprResult.asBoolean(), constExp.getType(), true); } else if (sqlTypeName == SqlTypeName.DATE) { - if (!constExp.getType().isNullable() && exprResult.isNull()) { + if (!constExp.getType().isNullable() && exprResult.isNumericNull()) { throw new IAE("Illegal DATE constant: %s", constExp); } @@ -95,7 +95,7 @@ public void reduce( ) ); } else if (sqlTypeName == SqlTypeName.TIMESTAMP) { - if (!constExp.getType().isNullable() && exprResult.isNull()) { + if (!constExp.getType().isNullable() && exprResult.isNumericNull()) { throw new IAE("Illegal TIMESTAMP constant: %s", constExp); } From 15000cbb2289f28a1d2857bd7f03e5af9444b38b Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 16 Mar 2018 03:38:27 +0530 Subject: [PATCH 11/38] review comments --- .../query/aggregation/last/DoubleLastAggregatorFactory.java | 2 +- .../main/java/io/druid/query/filter/SelectorDimFilter.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 8edc224d544c..8ce69723c278 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -149,7 +149,7 @@ public void aggregate(ByteBuffer buf, int position) long lastTime = buf.getLong(position); if (pair.lhs >= lastTime) { buf.putLong(position, pair.lhs); - buf.putDouble(position + Long.BYTES, pair.rhs); + buf.putDouble(position + Double.BYTES, pair.rhs); } } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 8c18a99ccbba..12c96f64764d 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -76,12 +76,12 @@ public byte[] getCacheKey() byte[] valueBytes = (value == null) ? new byte[]{} : StringUtils.toUtf8(value); byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - return ByteBuffer.allocate(6 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) + return ByteBuffer.allocate(5 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) .put(DimFilterUtils.SELECTOR_CACHE_ID) - .put(value == null ? (byte) 1 : (byte) 0) .put(DimFilterUtils.STRING_SEPARATOR) .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) + .put(value == null ? (byte) 1 : (byte) 0) .put(valueBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) From 8ae2c975815b2bf9a809515bff04092ead53dc9f Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 16 Mar 2018 04:10:21 +0530 Subject: [PATCH 12/38] review comment - cache numeric val in StringExprEval --- .../java/io/druid/math/expr/ExprEval.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 195b74dc2747..a6b8f1767eeb 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -235,6 +235,9 @@ public Expr toExpr() private static class StringExprEval extends ExprEval { + private volatile boolean numericValComputed = false; + private volatile Number numericVal; + private StringExprEval(@Nullable String value) { super(NullHandling.emptyToNullIfNeeded(value)); @@ -285,13 +288,26 @@ private Number asNumber() if (value == null) { return null; } + if (numericValComputed) { + return numericVal; + } + Number rv = null; Long v = GuavaUtils.tryParseLong(value); // Do NOT use ternary operator here, because it makes Java to convert Long to Double if (v != null) { - return v; + rv = v; } else { - return Doubles.tryParse(value); + rv = Doubles.tryParse(value); } + + numericVal = rv; + numericValComputed = true; + return rv; + } + + private void cache(Number number){ + numericVal = number; + numericValComputed = true; } @Override From 6c812081e53627b4310eb6cc2b523bb67bc650de Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 16 Mar 2018 10:05:39 +0530 Subject: [PATCH 13/38] fix checkstyle --- common/src/main/java/io/druid/math/expr/ExprEval.java | 6 ++++-- .../groupby/GroupByLimitPushDownMultiNodeMergeTest.java | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index a6b8f1767eeb..804b0b9d9454 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -83,7 +83,8 @@ public static ExprEval bestEffortOf(@Nullable Object val) return new StringExprEval(val == null ? null : String.valueOf(val)); } - @Nullable final T value; + @Nullable + final T value; private ExprEval(T value) { @@ -305,7 +306,8 @@ private Number asNumber() return rv; } - private void cache(Number number){ + private void cache(Number number) + { numericVal = number; numericValComputed = true; } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index ad8f783b7037..0ea560a263bf 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -93,7 +93,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; From d23956decb3f3cd260e186af75ad69106179ece0 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 28 Mar 2018 21:01:50 +0530 Subject: [PATCH 14/38] fix compilation --- ...penderatorDriverRealtimeIndexTaskTest.java | 30 +++++++++---------- .../common/task/RealtimeIndexTaskTest.java | 22 +++++++------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index 976526ef2e1d..c50164beba1c 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -355,8 +355,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(2, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -417,8 +417,8 @@ public void testLateData() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(2, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -482,8 +482,8 @@ public void testMaxRowsPerSegment() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2000, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(2000, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(2000, sumMetric(task, null, "met1")); + Assert.assertEquals(2000, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -550,14 +550,14 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); + Assert.assertEquals(2, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "met1")); if (NullHandling.replaceWithDefault()) { - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "met1")); } else { Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); } - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -681,8 +681,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -754,7 +754,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task2, null, "met1")); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -772,7 +772,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); + Assert.assertEquals(2, sumMetric(task2, null, "met1")); awaitHandoffs(); @@ -829,7 +829,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task1, null, "met1")); // Trigger graceful shutdown. task1.stopGracefully(); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index f4fc448c3431..3f0ecef4fa5c 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -354,8 +354,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(2, sumMetric(task, null, "rows")); + Assert.assertEquals(3, sumMetric(task, null, "met1")); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -423,15 +423,15 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(1, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); + Assert.assertEquals(1, sumMetric(task, null, "rows")); + Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); if (NullHandling.replaceWithDefault()) { - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); } else { Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); } - Assert.assertEquals(1, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(1, sumMetric(task, null, "met1")); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -557,8 +557,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "rows")); + Assert.assertEquals(3, sumMetric(task, null, "met1")); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -630,7 +630,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task2, null, "rows")); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -651,7 +651,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); + Assert.assertEquals(2, sumMetric(task2, null, "rows")); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -712,7 +712,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task1, null, "rows")); // Trigger graceful shutdown. task1.stopGracefully(); From d4f9b89023acf0e1494664a7574dc4e9af65934f Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 10 Apr 2018 02:27:03 +0530 Subject: [PATCH 15/38] review comments --- .../java/io/druid/math/expr/ExprEval.java | 15 +++------ .../sql/QuantileSqlAggregatorTest.java | 14 ++++---- .../namespace/UriExtractionNamespace.java | 8 ++--- .../main/java/io/druid/indexer/JobHelper.java | 7 ++-- .../io/druid/indexer/path/StaticPathSpec.java | 7 ++-- ...penderatorDriverRealtimeIndexTaskTest.java | 18 +++++----- java-util/pom.xml | 4 +++ .../io/druid/common/config/NullHandling.java | 0 .../config/NullValueHandlingConfig.java | 0 .../druid/java/util/common/StringUtils.java | 33 +++++++++++++++++++ .../java/util/common/parsers/ParserUtils.java | 10 ++---- .../util/http/client/NettyHttpClient.java | 6 +--- .../io/druid/query/DefaultQueryMetrics.java | 9 ++--- .../query/aggregation/AggregatorFactory.java | 2 +- .../query/aggregation/AggregatorUtil.java | 3 +- .../aggregation/LongMaxAggregatorFactory.java | 19 ++++------- .../aggregation/LongMinAggregatorFactory.java | 19 ++++------- .../aggregation/LongSumAggregatorFactory.java | 19 ++++------- .../NullableAggregateCombiner.java | 15 ++++++--- .../query/aggregation/NullableAggregator.java | 2 +- .../NullableAggregatorFactory.java | 2 +- .../aggregation/NullableBufferAggregator.java | 4 +-- ...alityAggregatorColumnSelectorStrategy.java | 2 +- ...alityAggregatorColumnSelectorStrategy.java | 2 +- ...alityAggregatorColumnSelectorStrategy.java | 2 +- .../first/FloatFirstAggregatorFactory.java | 2 +- .../first/LongFirstAggregatorFactory.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 2 +- .../post/DoubleGreatestPostAggregator.java | 5 ++- .../post/DoubleLeastPostAggregator.java | 5 ++- .../post/LongGreatestPostAggregator.java | 5 ++- .../post/LongLeastPostAggregator.java | 5 ++- .../query/extraction/LowerExtractionFn.java | 5 +-- .../query/extraction/MapLookupExtractor.java | 4 +-- .../extraction/StringFormatExtractionFn.java | 4 +-- .../query/extraction/UpperExtractionFn.java | 5 +-- .../io/druid/query/filter/InDimFilter.java | 18 +++------- .../druid/query/filter/SelectorDimFilter.java | 5 +-- .../io/druid/query/lookup/LookupConfig.java | 6 ++-- .../segment/filter/FilterPartitionTest.java | 6 ++-- .../query/dimension/LookupDimensionSpec.java | 6 +--- .../io/druid/query/lookup/LookupModule.java | 7 ++-- .../java/io/druid/server/QueryLifecycle.java | 11 ++----- .../druid/server/emitter/EmitterModule.java | 13 +++----- .../announcer/ListeningAnnouncerConfig.java | 7 ++-- .../dimension/LookupDimensionSpecTest.java | 6 ++-- .../coordinator/rules/LoadRuleTest.java | 3 -- 47 files changed, 150 insertions(+), 204 deletions(-) rename {common => java-util}/src/main/java/io/druid/common/config/NullHandling.java (100%) rename {common => java-util}/src/main/java/io/druid/common/config/NullValueHandlingConfig.java (100%) diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 804b0b9d9454..54cc27ef9765 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -236,8 +236,7 @@ public Expr toExpr() private static class StringExprEval extends ExprEval { - private volatile boolean numericValComputed = false; - private volatile Number numericVal; + private Number numericVal; private StringExprEval(@Nullable String value) { @@ -289,10 +288,11 @@ private Number asNumber() if (value == null) { return null; } - if (numericValComputed) { + if (numericVal != null) { + // Optimization for non-null case. return numericVal; } - Number rv = null; + Number rv; Long v = GuavaUtils.tryParseLong(value); // Do NOT use ternary operator here, because it makes Java to convert Long to Double if (v != null) { @@ -302,16 +302,9 @@ private Number asNumber() } numericVal = rv; - numericValComputed = true; return rv; } - private void cache(Number number) - { - numericVal = number; - numericValComputed = true; - } - @Override public boolean isNumericNull() { diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index a1ddda072f21..9b82b105ae07 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -313,14 +313,12 @@ public void testQuantileOnInnerQuery() throws Exception // Verify results final List results = plannerResult.run().toList(); - final List expectedResults = - NullHandling.replaceWithDefault() ? - ImmutableList.of( - new Object[]{7.0, 8.26386833190918} - ) : - ImmutableList.of( - new Object[]{5.25, 6.59091854095459} - ); + final List expectedResults; + if (NullHandling.replaceWithDefault()) { + expectedResults = ImmutableList.of(new Object[]{7.0, 8.26386833190918}); + } else { + expectedResults = ImmutableList.of(new Object[]{5.25, 6.59091854095459}); + } Assert.assertEquals(expectedResults.size(), results.size()); for (int i = 0; i < expectedResults.size(); i++) { Assert.assertArrayEquals(expectedResults.get(i), results.get(i)); diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index 7033a6e68b8e..8a6e9b3b3427 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap; import io.druid.guice.annotations.Json; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; import io.druid.java.util.common.jackson.JacksonUtils; import io.druid.java.util.common.parsers.CSVParser; @@ -395,15 +396,12 @@ public TSVFlatDataParser( Preconditions.checkNotNull(columns, "`columns` list required").size() > 1, "Must specify more than one column to have a key value pair" ); - //CHECKSTYLE.OFF: Regexp - // String.emptytoNull usage here is irrelevant to null handling of the data. final DelimitedParser delegate = new DelimitedParser( - Strings.emptyToNull(delimiter), - Strings.emptyToNull(listDelimiter), + StringUtils.emptyToNullNonDruidDataString(delimiter), + StringUtils.emptyToNullNonDruidDataString(listDelimiter), hasHeaderRow, skipHeaderRows ); - //CHECKSTYLE.ON: Regexp Preconditions.checkArgument( !(Strings.isNullOrEmpty(keyColumn) ^ Strings.isNullOrEmpty(valueColumn)), "Must specify both `keyColumn` and `valueColumn` or neither `keyColumn` nor `valueColumn`" diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 26b82be64b29..ccca7a450d1c 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -293,11 +293,8 @@ public static void injectSystemProperties(Job job) public static void injectDruidProperties(Configuration configuration, List listOfAllowedPrefix) { - //CHECKSTYLE.OFF: Regexp - // String.nulltoEmpty usage here is irrelevant to null handling of the data. - String mapJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.MAP_JAVA_OPTS)); - String reduceJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.REDUCE_JAVA_OPTS)); - //CHECKSTYLE.ON: regexp + String mapJavaOpts = StringUtils.nullToEmptyNonDruidDataString(configuration.get(MRJobConfig.MAP_JAVA_OPTS)); + String reduceJavaOpts = StringUtils.nullToEmptyNonDruidDataString(configuration.get(MRJobConfig.REDUCE_JAVA_OPTS)); for (String propName : System.getProperties().stringPropertyNames()) { for (String prefix : listOfAllowedPrefix) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java index 3bae9b68257f..2cfb1d164436 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java @@ -21,11 +21,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import io.druid.indexer.HadoopDruidIndexerConfig; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -126,10 +126,7 @@ public static void addToMultipleInputs( private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass) { Configuration conf = job.getConfiguration(); - //CHECKSTYLE.OFF: Regexp - // String.nulltoempty usage here is irrelevant to null handling of the data. - StringBuilder inputFormats = new StringBuilder(Strings.nullToEmpty(conf.get(MultipleInputs.DIR_FORMATS))); - //CHECKSTYLE.ON: Regexp + StringBuilder inputFormats = new StringBuilder(StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS))); String[] paths = Iterables.toArray(pathStrings, String.class); for (int i = 0; i < paths.length - 1; i++) { diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index c50164beba1c..d764c47e8b97 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -355,7 +355,7 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows")); Assert.assertEquals(3, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -417,7 +417,7 @@ public void testLateData() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows")); Assert.assertEquals(3, sumMetric(task, null, "met1")); awaitHandoffs(); @@ -550,12 +550,12 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "met1")); - Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows")); + Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); if (NullHandling.replaceWithDefault()) { - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "met1")); + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "metric1")); } else { - Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); + Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "metric1")); } Assert.assertEquals(3, sumMetric(task, null, "met1")); @@ -754,7 +754,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "met1")); + Assert.assertEquals(1, sumMetric(task2, null, "rows")); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -772,7 +772,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "met1")); + Assert.assertEquals(2, sumMetric(task2, null, "rows")); awaitHandoffs(); @@ -829,7 +829,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "met1")); + Assert.assertEquals(1, sumMetric(task1, null, "rows")); // Trigger graceful shutdown. task1.stopGracefully(); diff --git a/java-util/pom.xml b/java-util/pom.xml index 150c332a3ca7..41be402159cf 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -53,6 +53,10 @@ org.skife.config config-magic + + com.google.inject + guice + com.google.guava guava diff --git a/common/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java similarity index 100% rename from common/src/main/java/io/druid/common/config/NullHandling.java rename to java-util/src/main/java/io/druid/common/config/NullHandling.java diff --git a/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java b/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java similarity index 100% rename from common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java rename to java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index d073ef38b42f..125d55d9f131 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -20,6 +20,7 @@ package io.druid.java.util.common; import com.google.common.base.Charsets; +import com.google.common.base.Strings; import com.google.common.base.Throwables; import javax.annotation.Nullable; @@ -172,4 +173,36 @@ private static String removeChar(String s, char c, int firstOccurranceIndex) } return sb.toString(); } + + /** + * Returns the given string if it is non-null; the empty string otherwise. + * This method should only be used at places where null to empty conversion is + * irrelevant to null handling of the data. + * + * @param string the string to test and possibly return + * @return {@code string} itself if it is non-null; {@code ""} if it is null + */ + public static String nullToEmptyNonDruidDataString(@Nullable String string) + { + //CHECKSTYLE.OFF: Regexp + return Strings.nullToEmpty(string); + //CHECKSTYLE.ON: Regexp + } + + /** + * Returns the given string if it is nonempty; {@code null} otherwise. + * This method should only be used at places where null to empty conversion is + * irrelevant to null handling of the data. + * + * @param string the string to test and possibly return + * @return {@code string} itself if it is nonempty; {@code null} if it is + * empty or null + */ + @Nullable + public static String emptyToNullNonDruidDataString(@Nullable String string) + { + //CHECKSTYLE.OFF: Regexp + return Strings.emptyToNull(string); + //CHECKSTYLE.ON: Regexp + } } diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java index 6859c70c4786..bd062c41b9e2 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java @@ -21,8 +21,8 @@ import com.google.common.base.Function; import com.google.common.base.Splitter; -import com.google.common.base.Strings; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import org.joda.time.DateTimeZone; @@ -58,17 +58,13 @@ public static Function getMultiValueFunction( ) { return (input) -> { - //CHECKSTYLE.OFF: Regexp - // TODO: java-util does not depen on druid-common.NullHandling is not accessible here. - // Fix null semantics for CSV and TSV parsers. if (input != null && input.contains(listDelimiter)) { return StreamSupport.stream(listSplitter.split(input).spliterator(), false) - .map(Strings::emptyToNull) + .map(NullHandling::emptyToNullIfNeeded) .collect(Collectors.toList()); } else { - return Strings.emptyToNull(input); + return NullHandling.emptyToNullIfNeeded(input); } - //CHECKSTYLE.ON: Regexp }; } diff --git a/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java index a1eab6ef5898..712ba9b9830a 100644 --- a/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java +++ b/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java @@ -20,7 +20,6 @@ package io.druid.java.util.http.client; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Multimap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -140,10 +139,7 @@ public ListenableFuture go( } else { channel = channelFuture.getChannel(); } - //CHECKSTYLE.OFF: Regexp - // String.nullToEmpty usage here is irrelevant to null handling of the data. - final String urlFile = Strings.nullToEmpty(url.getFile()); - //CHECKSTYLE.ON: Regexp + final String urlFile = StringUtils.nullToEmptyNonDruidDataString(url.getFile()); final HttpRequest httpRequest = new DefaultHttpRequest( HttpVersion.HTTP_1_1, method, diff --git a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java index 0df370de62d8..e82a8c80a35c 100644 --- a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java @@ -21,11 +21,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.emitter.service.ServiceMetricEvent; -import io.druid.collections.bitmap.BitmapFactory; import io.druid.query.filter.Filter; import org.joda.time.Interval; @@ -118,10 +118,7 @@ public void duration(QueryType query) @Override public void queryId(QueryType query) { - //CHECKSTYLE.OFF: Regexp - // String.nullToEmpty usage here is irrelevant to null handling of the data. - setDimension(DruidMetrics.ID, Strings.nullToEmpty(query.getId())); - //CHECKSTYLE.ON: Regexp + setDimension(DruidMetrics.ID, StringUtils.nullToEmptyNonDruidDataString(query.getId())); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index ffe46400d868..a815d33066db 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -37,7 +37,7 @@ * max, sum of metric columns, or cardinality of dimension columns (see {@link * io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory}). * Implementations of {@link AggregatorFactory} which need to Support Nullable Aggregations are encouraged - * to extend {@link io.druid.query.aggregation.NullableAggregatorFactory}. + * to extend {@link NullableAggregatorFactory}. */ @ExtensionPoint public abstract class AggregatorFactory implements Cacheable diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 35bf50793756..1c5b5e80d30a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -34,7 +34,6 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.virtual.ExpressionSelectors; -import javax.annotation.Nullable; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -139,7 +138,7 @@ public static ColumnValueSelector makeColumnValueSelectorWithFloatDefault( final ExprMacroTable macroTable, final String fieldName, final String fieldExpression, - @Nullable final Float nullValue + final float nullValue ) { if (fieldName != null && fieldExpression == null) { diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index f6d8fedce9d7..ed3968cee931 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -74,7 +74,13 @@ public LongMaxAggregatorFactory(String name, String fieldName) @Override protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - return getLongColumnSelector(metricFactory); + return AggregatorUtil.makeColumnValueSelectorWithLongDefault( + metricFactory, + macroTable, + fieldName, + expression, + Long.MIN_VALUE + ); } @Override @@ -89,17 +95,6 @@ protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory return new LongMaxBufferAggregator(selector); } - private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.makeColumnValueSelectorWithLongDefault( - metricFactory, - macroTable, - fieldName, - expression, - Long.MIN_VALUE - ); - } - @Override public Comparator getComparator() { diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 1a311e9fbc7b..1a298e3647a8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -75,7 +75,13 @@ public LongMinAggregatorFactory(String name, String fieldName) @Override protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - return getLongColumnSelector(metricFactory); + return AggregatorUtil.makeColumnValueSelectorWithLongDefault( + metricFactory, + macroTable, + fieldName, + expression, + Long.MAX_VALUE + ); } @Override @@ -90,17 +96,6 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, C return new LongMinBufferAggregator(selector); } - private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.makeColumnValueSelectorWithLongDefault( - metricFactory, - macroTable, - fieldName, - expression, - Long.MAX_VALUE - ); - } - @Override public Comparator getComparator() { diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 09f5c77d7572..3e72429848cb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -74,7 +74,13 @@ public LongSumAggregatorFactory(String name, String fieldName) @Override protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) { - return getLongColumnSelector(metricFactory); + return AggregatorUtil.makeColumnValueSelectorWithLongDefault( + metricFactory, + macroTable, + fieldName, + expression, + 0L + ); } @Override @@ -89,17 +95,6 @@ protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory return new LongSumBufferAggregator(selector); } - private ColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.makeColumnValueSelectorWithLongDefault( - metricFactory, - macroTable, - fieldName, - expression, - 0L - ); - } - @Override public Comparator getComparator() { diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index 23eb789ad2c6..a1cd4a27aa54 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; @@ -26,7 +27,7 @@ /** * The result of a NullableAggregateCombiner will be null if all the values to be combined are null values or no values are combined at all. * If any of the value is non-null, the result would be the value of the delegate combiner. - * Note that the delegate combiner is not required to perform check for {@link NullableAggregateCombiner#isNull} on the columnValueSelector as only non-null values + * Note that the delegate combiner is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values * will be passed to the delegate combiner. */ public class NullableAggregateCombiner implements AggregateCombiner @@ -54,10 +55,14 @@ public void reset(ColumnValueSelector selector) @Override public void fold(ColumnValueSelector selector) { - if (isNullResult) { - reset(selector); - } else { - delegate.fold(selector); + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (isNullResult) { + isNullResult = false; + delegate.reset(selector); + } else { + delegate.fold(selector); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index dea9cdf4732e..c4ed13335be0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -26,7 +26,7 @@ /** * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. - * Note that the delegate aggregator is not required to perform check for {@link NullableAggregator#isNull} on the columnValueSelector as only non-null values + * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values * will be passed to the delegate aggregator. */ public class NullableAggregator implements Aggregator diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 4cda2951d21f..7d28c6d3fbb3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -95,7 +95,7 @@ protected abstract BufferAggregator factorizeBuffered( /** * Creates an {@link AggregateCombiner} to fold rollup aggregation results from serveral "rows" of different indexes during * index merging. AggregateCombiner implements the same logic as {@link #combine}, with the difference that it uses - * {@link io.druid.segment.ColumnValueSelector} and it's subinterfaces to get inputs and implements {@code + * {@link ColumnValueSelector} and it's subinterfaces to get inputs and implements {@code * ColumnValueSelector} to provide output. * * @see AggregateCombiner diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 6447f4d24671..a462328912bb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -27,17 +27,17 @@ /** * The result of a NullableBufferAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. - * Note that the delegate aggregator is not required to perform check for {@link NullableBufferAggregator#isNull(ByteBuffer, int)} on the columnValueSelector as only non-null values + * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values * will be passed to the delegate aggregator. */ public class NullableBufferAggregator implements BufferAggregator { private static final byte IS_NULL_BYTE = (byte) 0; private static final byte IS_NOT_NULL_BYTE = (byte) 1; + private final BufferAggregator delegate; private final BaseNullableColumnValueSelector selector; - public NullableBufferAggregator(BufferAggregator delegate, BaseNullableColumnValueSelector selector) { this.delegate = delegate; diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java index 3874c1e4250c..8f72f5138150 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java @@ -26,7 +26,7 @@ import io.druid.segment.BaseDoubleColumnValueSelector; /** - * if performance of this class appears to be a bottleneck for somebody, + * If performance of this class appears to be a bottleneck for somebody, * one simple way to improve it is to split it into two different classes, * one that is used when {@link NullHandling#replaceWithDefault()} is false, * and one - when it's true, moving this computation out of the tight loop diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java index 995667f94237..ffa9d7647277 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java @@ -26,7 +26,7 @@ import io.druid.segment.BaseFloatColumnValueSelector; /** - * if performance of this class appears to be a bottleneck for somebody, + * If performance of this class appears to be a bottleneck for somebody, * one simple way to improve it is to split it into two different classes, * one that is used when {@link NullHandling#replaceWithDefault()} is false, * and one - when it's true, moving this computation out of the tight loop diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java index 75dba24c4568..904de5b118f5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java @@ -26,7 +26,7 @@ import io.druid.segment.BaseLongColumnValueSelector; /** - * if performance of this class appears to be a bottleneck for somebody, + * If performance of this class appears to be a bottleneck for somebody, * one simple way to improve it is to split it into two different classes, * one that is used when {@link NullHandling#replaceWithDefault()} is false, * and one - when it's true, moving this computation out of the tight loop diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index ed67d345f7ad..4687b65cd2a0 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -188,7 +188,7 @@ public Object deserialize(Object object) @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? object : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 9554662efa63..cee1c18e178b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -181,7 +181,7 @@ public Object deserialize(Object object) @Nullable public Object finalizeComputation(@Nullable Object object) { - return object == null ? object : ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 8ce69723c278..8edc224d544c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -149,7 +149,7 @@ public void aggregate(ByteBuffer buf, int position) long lastTime = buf.getLong(position); if (pair.lhs >= lastTime) { buf.putLong(position, pair.lhs); - buf.putDouble(position + Double.BYTES, pair.rhs); + buf.putDouble(position + Long.BYTES, pair.rhs); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 7e0100028e8e..b77191727a35 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import io.druid.common.config.NullHandling; -import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -39,7 +38,7 @@ public class DoubleGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); + private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Number::doubleValue)); private final String name; private final List fields; @@ -80,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) > 0) { + if (nextVal != null && COMPARATOR.compare(nextVal, retVal) > 0) { retVal = nextVal.doubleValue(); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index 583328a4249c..dc9df62f9ba7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import io.druid.common.config.NullHandling; import io.druid.query.Queries; @@ -39,7 +38,7 @@ public class DoubleLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); + private static final Comparator COMPARATOR = Comparator.nullsLast(Comparator.comparingDouble(Number::doubleValue)); private final String name; private final List fields; @@ -80,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) < 0) { + if (nextVal != null && COMPARATOR.compare(nextVal, retVal) < 0) { retVal = nextVal.doubleValue(); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index 69f439aacb93..c2df9d90135b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import io.druid.common.config.NullHandling; -import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -39,7 +38,7 @@ public class LongGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); + private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Number::longValue)); private final String name; private final List fields; @@ -80,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) > 0) { + if (nextVal != null && COMPARATOR.compare(nextVal, retVal) > 0) { retVal = nextVal.longValue(); } } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index 01835ed2e34d..19063ba989ab 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import io.druid.common.config.NullHandling; import io.druid.query.Queries; @@ -39,7 +38,7 @@ public class LongLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); + private static final Comparator COMPARATOR = Comparator.nullsLast(Comparator.comparingLong(Number::longValue)); private final String name; private final List fields; @@ -80,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) < 0) { + if (nextVal != null && COMPARATOR.compare(nextVal, retVal) < 0) { retVal = nextVal.longValue(); } } diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index b0dddf121b73..90648d6a9e81 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Strings; import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; @@ -74,9 +73,7 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { - //CHECKSTYLE.OFF: Regexp - byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); - //CHECKSTYLE.ON: Regexp + byte[] localeBytes = StringUtils.toUtf8(StringUtils.nullToEmptyNonDruidDataString(localeString)); return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_LOWER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index ccdae8f0932f..49200df6ad4d 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -76,9 +76,7 @@ public List unapply(final String value) { @Override public boolean apply(@Nullable String key) { - //CHECKSTYLE.OFF: Regexp - return map.get(key).equals(Strings.nullToEmpty(value)); - //CHECKSTYLE.ON: Regexp + return map.get(key).equals(StringUtils.nullToEmptyNonDruidDataString(value)); } }).keySet()); diff --git a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java index ba21754774fe..954ea27f245b 100644 --- a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java @@ -107,9 +107,7 @@ public String apply(@Nullable String value) value = ""; } } - //CHECKSTYLE.OFF: Regexp - return Strings.emptyToNull(StringUtils.format(format, value)); - //CHECKSTYLE.ON: Regexp + return StringUtils.emptyToNullNonDruidDataString(StringUtils.format(format, value)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index ec2b914d3806..10a7c81835f0 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Strings; import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; @@ -73,9 +72,7 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { - //CHECKSTYLE.OFF: Regexp - byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); - //CHECKSTYLE.ON: Regexp + byte[] localeBytes = StringUtils.toUtf8(StringUtils.nullToEmptyNonDruidDataString(localeString)); return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_UPPER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 4e56d7eda92c..e6f76e222498 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Iterables; import com.google.common.collect.Range; @@ -118,9 +117,7 @@ public byte[] getCacheKey() if (value == null) { hasNull = true; } - //CHECKSTYLE.OFF: Regexp - valuesBytes[index] = StringUtils.toUtf8(Strings.nullToEmpty(value)); - //CHECKSTYLE.ON: Regexp + valuesBytes[index] = StringUtils.toUtf8(StringUtils.nullToEmptyNonDruidDataString(value)); valuesBytesSize += valuesBytes[index].length + 1; ++index; } @@ -167,9 +164,7 @@ private InDimFilter optimizeLookup() // We cannot do an unapply()-based optimization if the selector value // and the replaceMissingValuesWith value are the same, since we have to match on // all values that are not present in the lookup. - //CHECKSTYLE.OFF: Regexp - final String convertedValue = Strings.emptyToNull(value); - //CHECKSTYLE.ON: Regexp + final String convertedValue = NullHandling.emptyToNullIfNeeded(value); if (!exFn.isRetainMissingValue() && Objects.equals(convertedValue, exFn.getReplaceMissingValueWith())) { return this; } @@ -214,9 +209,7 @@ public RangeSet getDimensionRangeSet(String dimension) } RangeSet retSet = TreeRangeSet.create(); for (String value : values) { - //CHECKSTYLE.OFF: Regexp - retSet.add(Range.singleton(Strings.nullToEmpty(value))); - //CHECKSTYLE.ON: Regexp + retSet.add(Range.singleton(StringUtils.nullToEmptyNonDruidDataString(value))); } return retSet; } @@ -266,11 +259,10 @@ public String toString() if (extractionFn != null) { builder.append(")"); } - //CHECKSTYLE.OFF: Regexp + builder.append(" IN (") - .append(Joiner.on(", ").join(Iterables.transform(values, input -> Strings.nullToEmpty(input)))) + .append(Joiner.on(", ").join(Iterables.transform(values, input -> StringUtils.nullToEmptyNonDruidDataString(input)))) .append(")"); - //CHECKSTYLE.ON: Regexp return builder.toString(); } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 12c96f64764d..a0a4edcafe3b 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; @@ -190,9 +189,7 @@ public RangeSet getDimensionRangeSet(String dimension) return null; } RangeSet retSet = TreeRangeSet.create(); - //CHECKSTYLE.OFF: Regexp - retSet.add(Range.singleton(Strings.nullToEmpty(value))); - //CHECKSTYLE.ON: Regexp + retSet.add(Range.singleton(StringUtils.nullToEmptyNonDruidDataString(value))); return retSet; } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java index f5fef048e94f..1afce6a26aa2 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Strings; +import io.druid.java.util.common.StringUtils; import javax.validation.constraints.Min; import java.util.Objects; @@ -56,9 +56,7 @@ public LookupConfig( @JsonProperty("snapshotWorkingDir") String snapshotWorkingDir ) { - //CHECKSTYLE.OFF: Regexp - this.snapshotWorkingDir = Strings.nullToEmpty(snapshotWorkingDir); - //CHECKSTYLE.ON: Regexp + this.snapshotWorkingDir = StringUtils.nullToEmptyNonDruidDataString(snapshotWorkingDir); } public String getSnapshotWorkingDir() diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index e45bc5d0397c..a5b80d01784f 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -21,7 +21,6 @@ import com.google.common.base.Function; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.druid.common.config.NullHandling; @@ -33,6 +32,7 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Pair; +import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.JavaScriptExtractionFn; @@ -117,9 +117,7 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { - //CHECKSTYLE.OFF: Regexp - final String valueOrNull = Strings.emptyToNull(value); - //CHECKSTYLE.ON: Regexp + final String valueOrNull = StringUtils.emptyToNullNonDruidDataString(value); final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index cf1e9216cff9..299b947a441d 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -173,11 +173,7 @@ public byte[] getCacheKey() ? getLookup().getCacheKey() : StringUtils.toUtf8(name); byte[] outputNameBytes = StringUtils.toUtf8(outputName); - //CHECKSTYLE.OFF: Regexp - // String.nullToEmpty usage here is irrelevant to null handling of the data. - byte[] replaceWithBytes = StringUtils.toUtf8(Strings.nullToEmpty(replaceMissingValueWith)); - //CHECKSTYLE.ON: Regexp - + byte[] replaceWithBytes = StringUtils.toUtf8(StringUtils.nullToEmptyNonDruidDataString(replaceMissingValueWith)); return ByteBuffer.allocate(6 + dimensionBytes.length diff --git a/server/src/main/java/io/druid/query/lookup/LookupModule.java b/server/src/main/java/io/druid/query/lookup/LookupModule.java index f2b89733a187..93195f371ac8 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupModule.java +++ b/server/src/main/java/io/druid/query/lookup/LookupModule.java @@ -28,7 +28,6 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; @@ -48,6 +47,7 @@ import io.druid.guice.annotations.Self; import io.druid.guice.annotations.Smile; import io.druid.initialization.DruidModule; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.query.dimension.LookupDimensionSpec; import io.druid.query.expression.LookupExprMacro; @@ -260,14 +260,11 @@ public String getLookupTier() ); final String lookupTier = lookupTierIsDatasource ? dataSourceTaskIdHolder.getDataSource() : this.lookupTier; - //CHECKSTYLE.OFF: Regexp - // String.emptytoNull usage here is irrelevant to null handling of the data. return Preconditions.checkNotNull( - lookupTier == null ? DEFAULT_TIER : Strings.emptyToNull(lookupTier), + lookupTier == null ? DEFAULT_TIER : StringUtils.emptyToNullNonDruidDataString(lookupTier), "Cannot have empty lookup tier from %s", lookupTierIsDatasource ? "bound value" : LookupModule.PROPERTY_BASE ); - //CHECKSTYLE.ON: Regexp } public String getLookupKey() diff --git a/server/src/main/java/io/druid/server/QueryLifecycle.java b/server/src/main/java/io/druid/server/QueryLifecycle.java index bd07f7babd3c..bdf52736f564 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycle.java +++ b/server/src/main/java/io/druid/server/QueryLifecycle.java @@ -25,6 +25,7 @@ import io.druid.client.DirectDruidClient; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.SequenceWrapper; import io.druid.java.util.common.guava.Sequences; @@ -289,15 +290,12 @@ public void emitLogsAndMetrics( try { final long queryTimeNs = System.nanoTime() - startNs; - //CHECKSTYLE.OFF: Regexp - // String.nullToEmpty usage here is irrelevant to null handling of the data. QueryMetrics queryMetrics = DruidMetrics.makeRequestMetrics( queryMetricsFactory, toolChest, baseQuery, - Strings.nullToEmpty(remoteAddress) + StringUtils.nullToEmptyNonDruidDataString(remoteAddress) ); - //CHECKSTYLE.ON: Regexp queryMetrics.success(success); queryMetrics.reportQueryTime(queryTimeNs); @@ -330,17 +328,14 @@ public void emitLogsAndMetrics( statsMap.put("reason", e.toString()); } } - //CHECKSTYLE.OFF: Regexp - // String.nullToEmpty usage here is irrelevant to null handling of the data. requestLogger.log( new RequestLogLine( DateTimes.utc(startMs), - Strings.nullToEmpty(remoteAddress), + StringUtils.nullToEmptyNonDruidDataString(remoteAddress), baseQuery, new QueryStats(statsMap) ) ); - //CHECKSTYLE.ON: Regexp } catch (Exception ex) { log.error(ex, "Unable to log query [%s]!", baseQuery); diff --git a/server/src/main/java/io/druid/server/emitter/EmitterModule.java b/server/src/main/java/io/druid/server/emitter/EmitterModule.java index dfed77b1b2f0..9c3641bad976 100644 --- a/server/src/main/java/io/druid/server/emitter/EmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/EmitterModule.java @@ -19,7 +19,6 @@ package io.druid.server.emitter; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -34,14 +33,15 @@ import com.google.inject.multibindings.MapBinder; import com.google.inject.name.Named; import com.google.inject.name.Names; -import io.druid.java.util.emitter.EmittingLogger; -import io.druid.java.util.emitter.core.Emitter; -import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.guice.LazySingleton; import io.druid.guice.ManageLifecycle; import io.druid.guice.annotations.Self; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.server.DruidNode; import java.lang.annotation.Annotation; @@ -86,12 +86,9 @@ public void configure(Binder binder) ExtraServiceDimensions.class ); String version = getClass().getPackage().getImplementationVersion(); - //CHECKSTYLE.OFF: Regexp - // String.nullToempty usage here is irrelevant to null handling of the data. extraServiceDimensions .addBinding("version") - .toInstance(Strings.nullToEmpty(version)); // Version is null during `mvn test`. - //CHECKSTYLE.ON: Regexp + .toInstance(StringUtils.nullToEmptyNonDruidDataString(version)); // Version is null during `mvn test`. } @Provides diff --git a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java index f3db4d57c900..535b5032ee6f 100644 --- a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java +++ b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.inject.Inject; +import io.druid.java.util.common.StringUtils; import io.druid.server.initialization.ZkPathsConfig; import org.apache.curator.utils.ZKPaths; @@ -91,13 +91,10 @@ public String toString() */ public String getAnnouncementPath(String listenerName) { - //CHECKSTYLE.OFF: Regexp - // String.nullToempty usage here is irrelevant to null handling of the data. return ZKPaths.makePath( getListenersPath(), Preconditions.checkNotNull( - Strings.emptyToNull(listenerName), "Listener name cannot be null" + StringUtils.emptyToNullNonDruidDataString(listenerName), "Listener name cannot be null" ) ); - //CHECKSTYPE.OFF: Regexp } } diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index 9e18a0487e21..e22422965aa9 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -22,9 +22,9 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.druid.jackson.DefaultObjectMapper; +import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.lookup.LookupExtractor; @@ -162,9 +162,7 @@ public Object[] parametersForTestApply() public void testApply(DimensionSpec dimensionSpec, Map map) { for (Map.Entry entry : map.entrySet()) { - //CHECKSTYLE.OFF: Regexp - Assert.assertEquals(Strings.emptyToNull(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); - //CHECKSTYLE.ON: Regexp + Assert.assertEquals(StringUtils.emptyToNullNonDruidDataString(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); } } diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index 51200461a959..0ad081badaab 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -64,9 +64,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -//CHECKSTYLE.OFF: Regexp -//CHECKSTYLE.ON: Regexp - /** */ public class LoadRuleTest From ddae594c970c7fa612f84e127a40d0aeb635adc8 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 10 Apr 2018 21:59:44 +0530 Subject: [PATCH 16/38] review comments --- common/src/main/java/io/druid/math/expr/ExprEval.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 54cc27ef9765..344a5dab0303 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -126,7 +126,7 @@ private abstract static class NumericExprEval extends ExprEval private NumericExprEval(@Nullable Number value) { - super(value == null ? NullHandling.defaultDoubleValue() : value); + super(value); } @Override @@ -158,7 +158,7 @@ private static class DoubleExprEval extends NumericExprEval { private DoubleExprEval(@Nullable Number value) { - super(value); + super(value == null ? NullHandling.defaultDoubleValue() : value); } @Override @@ -198,7 +198,7 @@ private static class LongExprEval extends NumericExprEval { private LongExprEval(@Nullable Number value) { - super(value); + super(value == null ? NullHandling.defaultLongValue() : value); } @Override From c20c9307732cd1080da8029af3b3ad24a8f7a5ea Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 10 Apr 2018 22:05:57 +0530 Subject: [PATCH 17/38] review comment --- .../druid/query/aggregation/NullableAggregatorFactory.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 7d28c6d3fbb3..7eb8d1186912 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -26,9 +26,9 @@ import io.druid.segment.ColumnValueSelector; /** - * abstract class with functionality to wrap {@link Aggregator}, {@link BufferAggregator} and {@link AggregateCombiner} to make them Nullable. - * Implementations of {@link AggregatorFactory} which need to Support Nullable Aggregations are encouraged - * to extend this class. + * Abstract class with functionality to wrap {@link Aggregator}, {@link BufferAggregator} and {@link AggregateCombiner} + * to support nullable aggregations for SQL compatibility. Implementations of {@link AggregatorFactory} which need to + * Support Nullable Aggregations are encouraged to extend this class. */ @ExtensionPoint public abstract class NullableAggregatorFactory extends AggregatorFactory From a83c9331f48d7ddc3997b861dddb2a184d07fcf6 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 11 Apr 2018 01:48:01 +0530 Subject: [PATCH 18/38] review comments and make lookup null handling consistent --- .../io/druid/server/lookup/LoadingLookup.java | 21 ++++++++------- .../io/druid/server/lookup/PollingLookup.java | 26 ++++++++++++++----- .../io/druid/indexer/path/StaticPathSpec.java | 4 ++- .../post/DoubleGreatestPostAggregator.java | 6 ++++- .../post/DoubleLeastPostAggregator.java | 6 ++++- .../post/LongGreatestPostAggregator.java | 6 ++++- .../post/LongLeastPostAggregator.java | 6 ++++- .../query/extraction/MapLookupExtractor.java | 23 +++++++++++----- .../extraction/StringFormatExtractionFn.java | 2 +- .../io/druid/query/filter/InDimFilter.java | 6 ++++- .../query/lookup/LookupExtractionFn.java | 3 +-- .../druid/query/lookup/LookupExtractor.java | 8 +++--- .../segment/filter/FilterPartitionTest.java | 3 +-- .../dimension/LookupDimensionSpecTest.java | 4 +-- 14 files changed, 85 insertions(+), 39 deletions(-) diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java index 014b7b648805..f6bb7c0f2e1d 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java @@ -21,7 +21,6 @@ import com.google.common.base.Preconditions; - import io.druid.common.config.NullHandling; import io.druid.java.util.common.logger.Logger; import io.druid.query.lookup.LookupExtractor; @@ -64,15 +63,17 @@ public LoadingLookup( @Override - @Nullable - public String apply(final String key) + public String apply(@Nullable final String key) { - if (key == null) { + String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); + if (keyEquivalent == null) { + // keyEquivalent is null for SQL Compatible Null Behavior return null; } + final String presentVal; try { - presentVal = loadingCache.get(key, new ApplyCallable(key)); + presentVal = loadingCache.get(keyEquivalent, new ApplyCallable(keyEquivalent)); return NullHandling.emptyToNullIfNeeded(presentVal); } catch (ExecutionException e) { @@ -82,15 +83,17 @@ public String apply(final String key) } @Override - public List unapply(final String value) + public List unapply(@Nullable final String value) { - // null value maps to empty list - if (value == null) { + String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); + if (valueEquivalent == null) { + // valueEquivalent is null for SQL Compatible Null Behavior + // null value maps to empty list when SQL Compatible return Collections.EMPTY_LIST; } final List retList; try { - retList = reverseLoadingCache.get(value, new UnapplyCallable(value)); + retList = reverseLoadingCache.get(valueEquivalent, new UnapplyCallable(valueEquivalent)); return retList; } catch (ExecutionException e) { diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java index fb31d925aee0..f399719a1812 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java @@ -34,7 +34,7 @@ import io.druid.server.lookup.cache.polling.PollingCacheFactory; import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; +import java.util.Collections; import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -109,8 +109,13 @@ public void close() @Override @Nullable - public String apply(@NotNull String key) + public String apply(@Nullable String key) { + String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); + if (keyEquivalent == null) { + // keyEquivalent is null for SQL Compatible Null Behavior + return null; + } final CacheRefKeeper cacheRefKeeper = refOfCacheKeeper.get(); if (cacheRefKeeper == null) { throw new ISE("Cache reference is null WTF"); @@ -119,9 +124,9 @@ public String apply(@NotNull String key) try { if (cache == null) { // it must've been closed after swapping while I was getting it. Try again. - return this.apply(key); + return this.apply(keyEquivalent); } - return NullHandling.emptyToNullIfNeeded((String) cache.get(key)); + return NullHandling.emptyToNullIfNeeded((String) cache.get(keyEquivalent)); } finally { if (cacheRefKeeper != null && cache != null) { @@ -131,8 +136,15 @@ public String apply(@NotNull String key) } @Override - public List unapply(final String value) + public List unapply(@Nullable final String value) { + String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); + if (valueEquivalent == null) { + // valueEquivalent is null for SQL Compatible Null Behavior + // null value maps to empty list when SQL Compatible + return Collections.EMPTY_LIST; + } + CacheRefKeeper cacheRefKeeper = refOfCacheKeeper.get(); if (cacheRefKeeper == null) { throw new ISE("pollingLookup id [%s] is closed", id); @@ -141,9 +153,9 @@ public List unapply(final String value) try { if (cache == null) { // it must've been closed after swapping while I was getting it. Try again. - return this.unapply(value); + return this.unapply(valueEquivalent); } - return cache.getKeys(value); + return cache.getKeys(valueEquivalent); } finally { if (cacheRefKeeper != null && cache != null) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java index 2cfb1d164436..2642c8598af1 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java @@ -126,7 +126,9 @@ public static void addToMultipleInputs( private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass) { Configuration conf = job.getConfiguration(); - StringBuilder inputFormats = new StringBuilder(StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS))); + StringBuilder inputFormats = new StringBuilder( + StringUtils.nullToEmptyNonDruidDataString(conf.get(MultipleInputs.DIR_FORMATS)) + ); String[] paths = Iterables.toArray(pathStrings, String.class); for (int i = 0; i < paths.length - 1; i++) { diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index b77191727a35..7f74364e7269 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -80,7 +80,11 @@ public Object compute(Map values) Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. if (nextVal != null && COMPARATOR.compare(nextVal, retVal) > 0) { - retVal = nextVal.doubleValue(); + if (nextVal instanceof Double) { + retVal = (Double) nextVal; + } else { + retVal = nextVal.doubleValue(); + } } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index dc9df62f9ba7..7eff7073f2e7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -80,7 +80,11 @@ public Object compute(Map values) Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. if (nextVal != null && COMPARATOR.compare(nextVal, retVal) < 0) { - retVal = nextVal.doubleValue(); + if (nextVal instanceof Double) { + retVal = (Double) nextVal; + } else { + retVal = nextVal.doubleValue(); + } } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index c2df9d90135b..e1c9c3e43780 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -80,7 +80,11 @@ public Object compute(Map values) Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. if (nextVal != null && COMPARATOR.compare(nextVal, retVal) > 0) { - retVal = nextVal.longValue(); + if (nextVal instanceof Long) { + retVal = (Long) nextVal; + } else { + retVal = nextVal.longValue(); + } } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index 19063ba989ab..1a9ef10bf033 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -80,7 +80,11 @@ public Object compute(Map values) Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. if (nextVal != null && COMPARATOR.compare(nextVal, retVal) < 0) { - retVal = nextVal.longValue(); + if (nextVal instanceof Long) { + retVal = (Long) nextVal; + } else { + retVal = nextVal.longValue(); + } } } return retVal; diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index 49200df6ad4d..dcda1cda3f73 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -29,13 +29,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.lookup.LookupExtractor; import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -64,22 +65,32 @@ public Map getMap() @Nullable @Override - public String apply(@NotNull String val) + public String apply(@Nullable String key) { - return map.get(val); + String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); + if (keyEquivalent == null) { + // keyEquivalent is null for SQL Compatible Null Behavior + return null; + } + return NullHandling.emptyToNullIfNeeded(map.get(keyEquivalent)); } @Override - public List unapply(final String value) + public List unapply(@Nullable final String value) { + String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); + if (valueEquivalent == null) { + // valueEquivalent is null for SQL Compatible Null Behavior + // null value maps to empty list when SQL Compatible + return Collections.EMPTY_LIST; + } return Lists.newArrayList(Maps.filterKeys(map, new Predicate() { @Override public boolean apply(@Nullable String key) { - return map.get(key).equals(StringUtils.nullToEmptyNonDruidDataString(value)); + return map.get(key).equals(valueEquivalent); } }).keySet()); - } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java index 954ea27f245b..979fe96e9910 100644 --- a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java @@ -107,7 +107,7 @@ public String apply(@Nullable String value) value = ""; } } - return StringUtils.emptyToNullNonDruidDataString(StringUtils.format(format, value)); + return io.druid.common.config.NullHandling.emptyToNullIfNeeded(StringUtils.format(format, value)); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index e6f76e222498..39053315748d 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Iterables; import com.google.common.collect.Range; @@ -117,7 +118,10 @@ public byte[] getCacheKey() if (value == null) { hasNull = true; } - valuesBytes[index] = StringUtils.toUtf8(StringUtils.nullToEmptyNonDruidDataString(value)); + //CHECKSTYLE.OFF: Regexp + // Strings.nullToEmpty is safe to use here as we have encoded nullability in hasNull flag. + valuesBytes[index] = StringUtils.toUtf8(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp valuesBytesSize += valuesBytes[index].length + 1; ++index; } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index 57dc967e1af2..c62ad7fe196c 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Throwables; -import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionCacheHelper; import io.druid.query.extraction.FunctionalExtraction; @@ -54,7 +53,7 @@ public LookupExtractionFn( @Override public String apply(@Nullable String input) { - return NullHandling.emptyToNullIfNeeded(lookup.apply(NullHandling.nullToEmptyIfNeeded(input))); + return lookup.apply(input); } }, retainMissingValue, diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java index 1a487fab63d5..21ad0a054e9d 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java @@ -39,9 +39,9 @@ public abstract class LookupExtractor /** * Apply a particular lookup methodology to the input string * - * @param key The value to apply the lookup to. May not be null + * @param key The value to apply the lookup to. * - * @return The lookup, or null key cannot have the lookup applied to it and should be treated as missing. + * @return The lookup, or null when key is `null` or cannot have the lookup applied to it and should be treated as missing. */ @Nullable public abstract String apply(@Nullable String key); @@ -70,15 +70,15 @@ public Map applyAll(Iterable keys) * Provide the reverse mapping from a given value to a list of keys * * @param value the value to apply the reverse lookup - * Null and empty are considered to be the same value = nullToEmpty(value) * * @return the list of keys that maps to value or empty list. * Note that for the case of a none existing value in the lookup we have to cases either return an empty list OR list with null element. * returning an empty list implies that user want to ignore such a lookup value. * In the other hand returning a list with the null element implies user want to map the none existing value to the key null. + * Null value maps to empty list. */ - public abstract List unapply(String value); + public abstract List unapply(@Nullable String value); /** * @param values Iterable of values for which will perform reverse lookup diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index a5b80d01784f..bbf81e8a1b17 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -32,7 +32,6 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Pair; -import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.JavaScriptExtractionFn; @@ -117,7 +116,7 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { - final String valueOrNull = StringUtils.emptyToNullNonDruidDataString(value); + final String valueOrNull = NullHandling.emptyToNullIfNeeded(value); final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index e22422965aa9..a3b6282c2c93 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; -import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.MapLookupExtractor; import io.druid.query.lookup.LookupExtractor; @@ -162,7 +162,7 @@ public Object[] parametersForTestApply() public void testApply(DimensionSpec dimensionSpec, Map map) { for (Map.Entry entry : map.entrySet()) { - Assert.assertEquals(StringUtils.emptyToNullNonDruidDataString(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); + Assert.assertEquals(NullHandling.emptyToNullIfNeeded(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); } } From c22c43aefbaf435756dc334780e2fe045ad0e700 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 11 Apr 2018 02:35:16 +0530 Subject: [PATCH 19/38] more review comments and unit test fixes --- ...penderatorDriverRealtimeIndexTaskTest.java | 84 +++++++++++++------ .../common/task/RealtimeIndexTaskTest.java | 24 +++--- .../io/druid/common/config/NullHandling.java | 2 +- .../io/druid/query/filter/InDimFilter.java | 10 ++- .../druid/query/filter/SelectorDimFilter.java | 9 +- .../epinephelinae/RowBasedGrouperHelper.java | 24 +++--- .../groupby/strategy/GroupByStrategyV2.java | 1 + .../druid/segment/DoubleDimensionHandler.java | 7 +- .../druid/segment/FloatDimensionHandler.java | 7 +- .../druid/segment/LongDimensionHandler.java | 7 +- .../extraction/MapLookupExtractorTest.java | 17 +++- .../StringFormatExtractionFnTest.java | 6 +- .../filter/GetDimensionRangeSetTest.java | 50 ++++++++--- ...roupByLimitPushDownMultiNodeMergeTest.java | 1 + .../query/groupby/GroupByQueryRunnerTest.java | 2 +- .../epinephelinae/BufferArrayGrouperTest.java | 17 ++-- .../io/druid/segment/IndexMergerTestBase.java | 8 +- .../dimension/LookupDimensionSpecTest.java | 5 +- .../RegisteredLookupExtractionFnTest.java | 2 +- .../druid/sql/calcite/CalciteQueryTest.java | 7 +- 20 files changed, 205 insertions(+), 85 deletions(-) diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index 74103e0a08b8..022138f9d6da 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -223,7 +223,8 @@ public InputRow nextRow() @Override public Runnable commit() { - return () -> {}; + return () -> { + }; } @Override @@ -330,7 +331,8 @@ public void testHandoffTimeout() throws Exception // handoff would timeout, resulting in exception TaskStatus status = statusFuture.get(); - Assert.assertTrue(status.getErrorMsg().contains("java.util.concurrent.TimeoutException: Timeout waiting for task.")); + Assert.assertTrue(status.getErrorMsg() + .contains("java.util.concurrent.TimeoutException: Timeout waiting for task.")); } @Test(timeout = 60_000L) @@ -366,8 +368,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -428,8 +430,8 @@ public void testLateData() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -493,8 +495,8 @@ public void testMaxRowsPerSegment() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2000, sumMetric(task, null, "met1")); - Assert.assertEquals(2000, sumMetric(task, null, "met1")); + Assert.assertEquals(2000, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(2000, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -561,14 +563,14 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(2, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); if (NullHandling.replaceWithDefault()) { - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "metric1")); + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "metric1").longValue()); } else { Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "metric1")); } - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -623,7 +625,8 @@ public void testReportParseExceptionsOnBadMetric() throws Exception // Wait for the task to finish. TaskStatus status = statusFuture.get(); - Assert.assertTrue(status.getErrorMsg().contains("java.lang.RuntimeException: Max parse exceptions exceeded, terminating task...")); + Assert.assertTrue(status.getErrorMsg() + .contains("java.lang.RuntimeException: Max parse exceptions exceeded, terminating task...")); IngestionStatsAndErrorsTaskReportData reportData = getTaskReportData(); @@ -642,7 +645,15 @@ public void testNoReportParseExceptions() throws Exception { expectPublishedSegments(1); - final AppenderatorDriverRealtimeIndexTask task = makeRealtimeTask(null, TransformSpec.NONE, false, 0, true, null, 1); + final AppenderatorDriverRealtimeIndexTask task = makeRealtimeTask( + null, + TransformSpec.NONE, + false, + 0, + true, + null, + 1 + ); final ListenableFuture statusFuture = runTask(task); // Wait for firehose to show up, it starts off null. @@ -686,8 +697,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "met1")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -752,7 +763,18 @@ public void testMultipleParseExceptionsSuccess() throws Exception ImmutableMap.of("t", 1521251960729L, "dim1", "foo", "met1", "foo"), // Bad long dim- will count as processed, but bad dims will get default values - ImmutableMap.of("t", 1521251960729L, "dim1", "foo", "dimLong", "notnumber", "dimFloat", "notnumber", "met1", "foo"), + ImmutableMap.of( + "t", + 1521251960729L, + "dim1", + "foo", + "dimLong", + "notnumber", + "dimFloat", + "notnumber", + "met1", + "foo" + ), // Bad row- will be unparseable. ImmutableMap.of("dim1", "foo", "met1", 2.0, FAIL_DIM, "x"), @@ -777,8 +799,8 @@ public void testMultipleParseExceptionsSuccess() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(4, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(4, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -854,7 +876,18 @@ public void testMultipleParseExceptionsFailure() throws Exception ImmutableMap.of("t", 1521251960729L, "dim1", "foo", "met1", "foo"), // Bad long dim- will count as processed, but bad dims will get default values - ImmutableMap.of("t", 1521251960729L, "dim1", "foo", "dimLong", "notnumber", "dimFloat", "notnumber", "met1", "foo"), + ImmutableMap.of( + "t", + 1521251960729L, + "dim1", + "foo", + "dimLong", + "notnumber", + "dimFloat", + "notnumber", + "met1", + "foo" + ), // Bad row- will be unparseable. ImmutableMap.of("dim1", "foo", "met1", 2.0, FAIL_DIM, "x"), @@ -945,7 +978,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows")); + Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -963,7 +996,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows")); + Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); awaitHandoffs(); @@ -1020,7 +1053,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(publishedSegments); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows")); + Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); // Trigger graceful shutdown. task1.stopGracefully(); @@ -1139,7 +1172,8 @@ public void testRestoreCorruptData() throws Exception ) ); Assert.assertEquals(expectedMetrics, reportData.getRowStats()); - Assert.assertTrue(status.getErrorMsg().contains("java.lang.IllegalArgumentException\n\tat java.nio.Buffer.position")); + Assert.assertTrue(status.getErrorMsg() + .contains("java.lang.IllegalArgumentException\n\tat java.nio.Buffer.position")); } } @@ -1465,7 +1499,7 @@ public List getLocations() ); } - public long sumMetric(final Task task, final DimFilter filter, final String metric) + public Long sumMetric(final Task task, final DimFilter filter, final String metric) { // Do a query. TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index 1b2c566fb3ee..8c933db927bc 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -353,8 +353,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -422,15 +422,15 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(1, sumMetric(task, null, "rows")); - Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); + Assert.assertEquals(1, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); if (NullHandling.replaceWithDefault()) { - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); } else { Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); } - Assert.assertEquals(1, sumMetric(task, null, "met1")); + Assert.assertEquals(1, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -545,8 +545,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -618,7 +618,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows")); + Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -639,7 +639,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows")); + Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -700,7 +700,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows")); + Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); // Trigger graceful shutdown. task1.stopGracefully(); @@ -1086,7 +1086,7 @@ public List getLocations() return toolboxFactory.build(task); } - public long sumMetric(final Task task, final DimFilter filter, final String metric) + public Long sumMetric(final Task task, final DimFilter filter, final String metric) { // Do a query. TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() diff --git a/java-util/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java index 762f8213214f..bdb9ac7faf7c 100644 --- a/java-util/src/main/java/io/druid/common/config/NullHandling.java +++ b/java-util/src/main/java/io/druid/common/config/NullHandling.java @@ -50,7 +50,7 @@ public class NullHandling */ @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( - Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) + false ); public static boolean replaceWithDefault() diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 39053315748d..7cbd33e73acf 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -213,7 +213,15 @@ public RangeSet getDimensionRangeSet(String dimension) } RangeSet retSet = TreeRangeSet.create(); for (String value : values) { - retSet.add(Range.singleton(StringUtils.nullToEmptyNonDruidDataString(value))); + String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); + if (valueEquivalent == null) { + // Case when SQL compatible null handling is enabled + // Range.singleton(null) is invalid, so use the fact that + // only null values are less than empty string. + retSet.add(Range.lessThan("")); + } else { + retSet.add(Range.singleton(valueEquivalent)); + } } return retSet; } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index a0a4edcafe3b..fc73f3a485ae 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -189,7 +189,14 @@ public RangeSet getDimensionRangeSet(String dimension) return null; } RangeSet retSet = TreeRangeSet.create(); - retSet.add(Range.singleton(StringUtils.nullToEmptyNonDruidDataString(value))); + String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); + if (valueEquivalent == null) { + // Case when SQL compatible null handling is enabled + // Nulls are less than empty String in segments + retSet.add(Range.lessThan("")); + } else { + retSet.add(Range.singleton(valueEquivalent)); + } return retSet; } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index f4a1127c823e..5dac2b2b9cf4 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -1549,13 +1549,13 @@ private class LongRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> Longs.compare( - lhsBuffer.getLong(lhsPosition + keyBufferPosition), - rhsBuffer.getLong(rhsPosition + keyBufferPosition) + lhsBuffer.getLong(lhsPosition + keyBufferPosition + Byte.BYTES), + rhsBuffer.getLong(rhsPosition + keyBufferPosition + Byte.BYTES) ); } else { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - long lhs = lhsBuffer.getLong(lhsPosition + keyBufferPosition); - long rhs = rhsBuffer.getLong(rhsPosition + keyBufferPosition); + long lhs = lhsBuffer.getLong(lhsPosition + keyBufferPosition + Byte.BYTES); + long rhs = rhsBuffer.getLong(rhsPosition + keyBufferPosition + Byte.BYTES); return stringComparator.compare(String.valueOf(lhs), String.valueOf(rhs)); }; @@ -1618,13 +1618,13 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> Float.compare( - lhsBuffer.getFloat(lhsPosition + keyBufferPosition), - rhsBuffer.getFloat(rhsPosition + keyBufferPosition) + lhsBuffer.getFloat(lhsPosition + keyBufferPosition + Byte.BYTES), + rhsBuffer.getFloat(rhsPosition + keyBufferPosition + Byte.BYTES) ); } else { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - float lhs = lhsBuffer.getFloat(lhsPosition + keyBufferPosition); - float rhs = rhsBuffer.getFloat(rhsPosition + keyBufferPosition); + float lhs = lhsBuffer.getFloat(lhsPosition + keyBufferPosition + Byte.BYTES); + float rhs = rhsBuffer.getFloat(rhsPosition + keyBufferPosition + Byte.BYTES); return stringComparator.compare(String.valueOf(lhs), String.valueOf(rhs)); }; } @@ -1686,13 +1686,13 @@ private class DoubleRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> Double.compare( - lhsBuffer.getDouble(lhsPosition + keyBufferPosition), - rhsBuffer.getDouble(rhsPosition + keyBufferPosition) + lhsBuffer.getDouble(lhsPosition + keyBufferPosition + Byte.BYTES), + rhsBuffer.getDouble(rhsPosition + keyBufferPosition + Byte.BYTES) ); } else { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - double lhs = lhsBuffer.getDouble(lhsPosition + keyBufferPosition); - double rhs = rhsBuffer.getDouble(rhsPosition + keyBufferPosition); + double lhs = lhsBuffer.getDouble(lhsPosition + keyBufferPosition + Byte.BYTES); + double rhs = rhsBuffer.getDouble(rhsPosition + keyBufferPosition + Byte.BYTES); return stringComparator.compare(String.valueOf(lhs), String.valueOf(rhs)); }; } diff --git a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java index b4070425c3ea..9e1d16926f7e 100644 --- a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java +++ b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java @@ -235,6 +235,7 @@ protected BinaryFn createMergeFn(Query queryParam) context.put(CTX_KEY_FUDGE_TIMESTAMP, String.valueOf(fudgeTimestamp.getMillis())); } context.put(CTX_KEY_OUTERMOST, false); + // the having spec shouldn't be passed down, so we need to convey the existing limit push down status context.put(GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown()); final GroupByQuery newQuery = new GroupByQuery( diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java b/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java index aac925fce7d7..aef471734e52 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionHandler.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.GenericColumn; @@ -99,6 +100,10 @@ public Closeable getSubColumn(Column column) @Override public Double getEncodedKeyComponentFromColumn(Closeable column, int currRow) { - return ((GenericColumn) column).getDoubleSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) column; + if (NullHandling.sqlCompatible() && genericColumn.isNull(currRow)) { + return null; + } + return genericColumn.getDoubleSingleValueRow(currRow); } } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java index 7e63f55044c8..7d84457ac094 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.GenericColumn; @@ -99,6 +100,10 @@ public Closeable getSubColumn(Column column) @Override public Float getEncodedKeyComponentFromColumn(Closeable column, int currRow) { - return ((GenericColumn) column).getFloatSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) column; + if (NullHandling.sqlCompatible() && genericColumn.isNull(currRow)) { + return null; + } + return genericColumn.getFloatSingleValueRow(currRow); } } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java index 3a27c0adf302..106acc799767 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.GenericColumn; @@ -99,6 +100,10 @@ public Closeable getSubColumn(Column column) @Override public Long getEncodedKeyComponentFromColumn(Closeable column, int currRow) { - return ((GenericColumn) column).getLongSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) column; + if (NullHandling.sqlCompatible() && genericColumn.isNull(currRow)) { + return null; + } + return genericColumn.getLongSingleValueRow(currRow); } } diff --git a/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java b/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java index 870812dcac4b..5f841f7513a3 100644 --- a/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -39,9 +40,19 @@ public void testUnApply() { Assert.assertEquals(Arrays.asList("foo"), fn.unapply("bar")); Assert.assertEquals(Sets.newHashSet("null", "empty String"), Sets.newHashSet(fn.unapply(""))); - Assert.assertEquals("Null value should be equal to empty string", - Sets.newHashSet("null", "empty String"), - Sets.newHashSet(fn.unapply((String) null))); + if (NullHandling.sqlCompatible()) { + Assert.assertEquals( + "Null value should be equal to empty list", + Sets.newHashSet(), + Sets.newHashSet(fn.unapply((String) null)) + ); + } else { + Assert.assertEquals( + "Null value should be equal to empty string", + Sets.newHashSet("null", "empty String"), + Sets.newHashSet(fn.unapply((String) null)) + ); + } Assert.assertEquals(Sets.newHashSet(""), Sets.newHashSet(fn.unapply("empty_string"))); Assert.assertEquals("not existing value returns empty list", Collections.EMPTY_LIST, fn.unapply("not There")); } diff --git a/processing/src/test/java/io/druid/query/extraction/StringFormatExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/StringFormatExtractionFnTest.java index 43289c3ddc31..b2248f72d56f 100644 --- a/processing/src/test/java/io/druid/query/extraction/StringFormatExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/StringFormatExtractionFnTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -55,7 +56,10 @@ public void testApplyNull2() { String test = null; Assert.assertEquals("null", format("%s", "nullString").apply(test)); - Assert.assertNull(format("%s", "emptyString").apply(test)); + Assert.assertEquals( + NullHandling.emptyToNullIfNeeded(""), + format("%s", "emptyString").apply(test) + ); Assert.assertNull(format("%s", "returnNull").apply(test)); } diff --git a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java index 1ba026c01b9b..a72b2b68313a 100644 --- a/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java +++ b/processing/src/test/java/io/druid/query/filter/GetDimensionRangeSetTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableRangeSet; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.Intervals; import io.druid.js.JavaScriptConfig; import io.druid.query.extraction.IdentityExtractionFn; @@ -60,7 +61,8 @@ public class GetDimensionRangeSetTest ); private final DimFilter other1 = new RegexDimFilter("someDim", "pattern", null); private final DimFilter other2 = new JavaScriptDimFilter("someOtherDim", "function(x) { return x }", null, - JavaScriptConfig.getEnabledInstance()); + JavaScriptConfig.getEnabledInstance() + ); private final DimFilter other3 = new SearchQueryDimFilter("dim", new ContainsSearchQuerySpec("a", true), null); private final DimFilter interval1 = new IntervalDimFilter( @@ -91,13 +93,13 @@ public void testSimpleFilter() Assert.assertEquals(expected1, selector1.getDimensionRangeSet("dim1")); Assert.assertNull(selector1.getDimensionRangeSet("dim2")); - RangeSet expected2 = rangeSet(point("")); + RangeSet expected2 = rangeSet(point(null)); Assert.assertEquals(expected2, selector5.getDimensionRangeSet("dim1")); RangeSet expected3 = rangeSet(ImmutableList.of(point("testing"), point("this"), point("filter"), point("tillend"))); Assert.assertEquals(expected3, in1.getDimensionRangeSet("dim1")); - RangeSet expected4 = rangeSet(ImmutableList.of(point("null"), point(""))); + RangeSet expected4 = rangeSet(ImmutableList.of(point("null"), point(null))); Assert.assertEquals(expected4, in3.getDimensionRangeSet("dim1")); RangeSet expected5 = ImmutableRangeSet.of(Range.closed("from", "to")); @@ -146,12 +148,13 @@ public void testAndFilter() public void testOrFilter() { DimFilter or1 = new OrDimFilter(ImmutableList.of(selector1, selector2, selector5)); - RangeSet expected1 = rangeSet(ImmutableList.of(point(""), point("a"), point("z"))); + RangeSet expected1 = rangeSet(ImmutableList.of(point(null), point("a"), point("z"))); Assert.assertEquals(expected1, or1.getDimensionRangeSet("dim1")); DimFilter or2 = new OrDimFilter(ImmutableList.of(selector5, in1, in3)); RangeSet expected2 = rangeSet(ImmutableList.of(point("testing"), point("this"), point("filter"), point("tillend"), - point("null"), point(""))); + point("null"), point(null) + )); Assert.assertEquals(expected2, or2.getDimensionRangeSet("dim1")); DimFilter or3 = new OrDimFilter(ImmutableList.of(bound1, bound2, bound3)); @@ -162,11 +165,13 @@ public void testOrFilter() Assert.assertNull(or4.getDimensionRangeSet("dim2")); DimFilter or5 = new OrDimFilter(ImmutableList.of(or1, or2, bound1)); - RangeSet expected5 = rangeSet(ImmutableList.of(point(""), point("a"), point("filter"), Range.closed("from", "to"), - point("z"))); + RangeSet expected5 = rangeSet(ImmutableList.of(point(null), point("a"), point("filter"), Range.closed("from", "to"), + point("z") + )); Assert.assertEquals(expected5, or5.getDimensionRangeSet("dim1")); } + @Test public void testNotFilter() { @@ -176,15 +181,28 @@ public void testNotFilter() Assert.assertNull(not1.getDimensionRangeSet("dim2")); DimFilter not2 = new NotDimFilter(in3); - RangeSet expected2 = rangeSet(ImmutableList.of(Range.lessThan(""), Range.open("", "null"), Range.greaterThan("null"))); - Assert.assertEquals(expected2, not2.getDimensionRangeSet("dim1")); + if (NullHandling.sqlCompatible()) { + // Empty string is included when != null for SQL Compatible case + RangeSet expected2 = rangeSet(ImmutableList.of( + Range.closedOpen("", "null"), + Range.greaterThan("null") + )); + Assert.assertEquals(expected2, not2.getDimensionRangeSet("dim1")); + } else { + RangeSet expected2 = rangeSet(ImmutableList.of( + Range.lessThan(""), + Range.open("", "null"), + Range.greaterThan("null") + )); + Assert.assertEquals(expected2, not2.getDimensionRangeSet("dim1")); + } DimFilter not3 = new NotDimFilter(bound1); RangeSet expected3 = rangeSet(ImmutableList.of(Range.lessThan("from"), Range.greaterThan("to"))); Assert.assertEquals(expected3, not3.getDimensionRangeSet("dim1")); DimFilter not4 = new NotDimFilter(not2); - RangeSet expected4 = rangeSet(ImmutableList.of(point(""), point("null"))); + RangeSet expected4 = rangeSet(ImmutableList.of(point(null), point("null"))); Assert.assertEquals(expected4, not4.getDimensionRangeSet("dim1")); DimFilter or1 = new OrDimFilter(ImmutableList.of(selector1, selector2, bound1, bound3)); @@ -203,7 +221,8 @@ public void testNotFilter() DimFilter and1 = new AndDimFilter(ImmutableList.of(in1, bound1, bound2)); DimFilter not7 = new NotDimFilter(and1); RangeSet expected7 = rangeSet(ImmutableList.of(Range.lessThan("testing"), Range.open("testing", "this"), - Range.open("this", "tillend"), Range.greaterThan("tillend"))); + Range.open("this", "tillend"), Range.greaterThan("tillend") + )); Assert.assertEquals(expected7, not7.getDimensionRangeSet("dim1")); Assert.assertNull(not7.getDimensionRangeSet("dim2")); @@ -216,6 +235,15 @@ public void testNotFilter() private static Range point(String s) { + if (s == null) { + if (NullHandling.sqlCompatible()) { + // Range.singleton(null) is invalid + return Range.lessThan(""); + } else { + // For non-sql compatible case, null and "" are equivalent + return Range.singleton(""); + } + } return Range.singleton(s); } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index 5cb158bc5314..f888046ae2fa 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -93,6 +93,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.File; diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 122711fdc2b4..5894029e0230 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -7482,7 +7482,7 @@ public void testGroupByWithAllFiltersOnNullDimsWithExtractionFns() extractionMap.put(null, "EMPTY"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn extractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn extractionFn = new LookupExtractionFn(mapLookupExtractor, false, "EMPTY", true, true); String jsFn = "function(x) { return(x === 'EMPTY') }"; List superFilterList = new ArrayList<>(); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java index acdc2c406047..c55fbf2b0afd 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; @@ -90,17 +91,23 @@ private BufferArrayGrouper newGrouper( @Test public void testRequiredBufferCapacity() { - int[] cardinalityArray = new int[] {1, 10, Integer.MAX_VALUE - 1}; - AggregatorFactory[] aggregatorFactories = new AggregatorFactory[] { + int[] cardinalityArray = new int[]{1, 10, Integer.MAX_VALUE - 1}; + AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{ new LongSumAggregatorFactory("sum", "sum") }; - - long[] requiredSizes = new long[] {17, 90, 16911433721L}; + long[] requiredSizes; + if (NullHandling.sqlCompatible()) { + // We need additional size to store nullability information. + requiredSizes = new long[]{19, 101, 19058917368L}; + } else { + requiredSizes = new long[]{17, 90, 16911433721L}; + } for (int i = 0; i < cardinalityArray.length; i++) { Assert.assertEquals(requiredSizes[i], BufferArrayGrouper.requiredBufferCapacity( cardinalityArray[i], - aggregatorFactories)); + aggregatorFactories + )); } } } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 576a152c1420..87404f534b3c 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -2178,9 +2178,13 @@ public void testMergeNumericDims() throws Exception Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames())); Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new Object[]{0L, 0.0f, new int[]{2}}, boatList.get(0).getDims()); + Assert.assertArrayEquals( + new Object[]{ + NullHandling.defaultLongValue(), + NullHandling.defaultFloatValue(), + new int[]{2} + }, boatList.get(0).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(0).getMetrics()); - Assert.assertArrayEquals(new Object[]{72L, 60000.789f, new int[]{3}}, boatList.get(1).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(0).getMetrics()); diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index a3b6282c2c93..8ff7137fb48a 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -31,6 +31,7 @@ import io.druid.query.lookup.LookupExtractorFactoryContainer; import io.druid.query.lookup.LookupReferencesManager; import io.druid.query.lookup.MapLookupExtractorFactory; +import io.druid.segment.TestHelper; import junitparams.JUnitParamsRunner; import junitparams.Parameters; import org.easymock.EasyMock; @@ -127,11 +128,11 @@ public Object[] parametersForTestApply() }, new Object[]{ new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, null, null, null, true), - ImmutableMap.of("not there", "") + TestHelper.createExpectedMap("not there", null) }, new Object[]{ new LookupDimensionSpec("dimName", "outputName", null, false, null, "lookupName", LOOKUP_REF_MANAGER, true), - ImmutableMap.of("not there", "") + TestHelper.createExpectedMap("not there", null) }, new Object[]{ new LookupDimensionSpec("dimName", "outputName", MAP_LOOKUP_EXTRACTOR, false, "Missing_value", null, null, diff --git a/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java b/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java index 0e990d89e37e..2643f31aa556 100644 --- a/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java +++ b/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java @@ -66,7 +66,7 @@ public void testSimpleDelegation() Assert.assertEquals(false, fn.isInjective()); Assert.assertEquals(ExtractionFn.ExtractionType.MANY_TO_ONE, fn.getExtractionType()); - for (String orig : Arrays.asList("", "foo", "bat")) { + for (String orig : Arrays.asList(null, "foo", "bat")) { Assert.assertEquals(LOOKUP_EXTRACTOR.apply(orig), fn.apply(orig)); } Assert.assertEquals("not in the map", fn.apply("not in the map")); diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 845c050c36c2..ed09821f65ef 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -1591,7 +1591,6 @@ public void testPruneDeadAggregatorsThroughHaving() throws Exception @Test public void testGroupByCaseWhen() throws Exception { - String nullValue = NullHandling.replaceWithDefault() ? "" : null; testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -1632,7 +1631,7 @@ public void testGroupByCaseWhen() throws Exception .build() ), ImmutableList.of( - new Object[]{nullValue, 2L}, + new Object[]{NullHandling.defaultStringValue(), 2L}, new Object[]{"match-cnt", 1L}, new Object[]{"match-m1 ", 3L} ) @@ -1656,7 +1655,7 @@ public void testGroupByCaseWhenOfTripleAnd() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "case_searched(((\"m1\" > 1) && (\"m1\" < 5) && (\"cnt\" == 1)),'x','')", + "case_searched(((\"m1\" > 1) && (\"m1\" < 5) && (\"cnt\" == 1)),'x',null)", ValueType.STRING ) ) @@ -1666,7 +1665,7 @@ public void testGroupByCaseWhenOfTripleAnd() throws Exception .build() ), ImmutableList.of( - new Object[]{"", 3L}, + new Object[]{NullHandling.defaultStringValue(), 3L}, new Object[]{"x", 3L} ) ); From b095f45e04f405062b84690db29fccd793b51e34 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 13 Apr 2018 02:44:06 +0530 Subject: [PATCH 20/38] set default useDefaultValuesForNull to true --- .../src/main/java/io/druid/common/config/NullHandling.java | 2 +- .../java/io/druid/common/config/NullValueHandlingConfig.java | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/java-util/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java index bdb9ac7faf7c..762f8213214f 100644 --- a/java-util/src/main/java/io/druid/common/config/NullHandling.java +++ b/java-util/src/main/java/io/druid/common/config/NullHandling.java @@ -50,7 +50,7 @@ public class NullHandling */ @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( - false + Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) ); public static boolean replaceWithDefault() diff --git a/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java b/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java index fdd1638b510e..2398f009ff67 100644 --- a/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java +++ b/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java @@ -31,9 +31,7 @@ public class NullValueHandlingConfig @JsonCreator public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) { - this.useDefaultValuesForNull = useDefaultValuesForNull == null - ? true - : useDefaultValuesForNull; + this.useDefaultValuesForNull = useDefaultValuesForNull; } public boolean isUseDefaultValuesForNull() From eddd435fce7ff6b7b3d9a605fdadf97660eeaeef Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 13 Apr 2018 02:46:58 +0530 Subject: [PATCH 21/38] fix checkstyle --- .../query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index f888046ae2fa..5cb158bc5314 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -93,7 +93,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.File; From 8bc95aa798a1fd476a6c9455df453ae0a775f8e1 Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 13 Apr 2018 03:09:06 +0530 Subject: [PATCH 22/38] revert unwanted change --- .../java/io/druid/common/config/NullValueHandlingConfig.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java b/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java index 2398f009ff67..fdd1638b510e 100644 --- a/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java +++ b/java-util/src/main/java/io/druid/common/config/NullValueHandlingConfig.java @@ -31,7 +31,9 @@ public class NullValueHandlingConfig @JsonCreator public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) { - this.useDefaultValuesForNull = useDefaultValuesForNull; + this.useDefaultValuesForNull = useDefaultValuesForNull == null + ? true + : useDefaultValuesForNull; } public boolean isUseDefaultValuesForNull() From 04199c3abb6e68b26819dca553cf270539c9626c Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 13 Apr 2018 03:09:45 +0530 Subject: [PATCH 23/38] fix comment --- .../src/main/java/io/druid/common/config/NullHandling.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-util/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java index 762f8213214f..c1ee4dd3ab44 100644 --- a/java-util/src/main/java/io/druid/common/config/NullHandling.java +++ b/java-util/src/main/java/io/druid/common/config/NullHandling.java @@ -46,7 +46,7 @@ public class NullHandling * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. * See io.druid.guice.NullHandlingModule for details. * It does not take effect in all unit tests since we don't use Guice Injection. - * For tests default system property is supposed to be used only in tests + * default system property is supposed to be used only in tests */ @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( From a7c88a9fa750def7a560790701e91d5d6808641e Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 13 Apr 2018 21:27:56 +0530 Subject: [PATCH 24/38] cosmetic changes --- .../src/main/java/io/druid/query/filter/InDimFilter.java | 1 + .../main/java/io/druid/query/filter/SelectorDimFilter.java | 3 +++ .../java/io/druid/segment/filter/FilterPartitionTest.java | 1 + .../java/io/druid/query/dimension/LookupDimensionSpec.java | 1 + .../io/druid/query/dimension/LookupDimensionSpecTest.java | 5 ++++- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index 7cbd33e73acf..e1ac4e4cff9d 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -60,6 +60,7 @@ public class InDimFilter implements DimFilter // Hashing threshold is not applied to String for now, String still uses ImmutableSortedSet public static final int NUMERIC_HASHING_THRESHOLD = 16; + // values can contain `null` string. private final SortedSet values; private final String dimension; private final ExtractionFn extractionFn; diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index fc73f3a485ae..060909324f1f 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -36,6 +36,7 @@ import io.druid.segment.filter.DimensionPredicateFilter; import io.druid.segment.filter.SelectorFilter; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Objects; @@ -45,6 +46,8 @@ public class SelectorDimFilter implements DimFilter { private final String dimension; + + @Nullable private final String value; private final ExtractionFn extractionFn; diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index bbf81e8a1b17..cb44cf27afff 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -107,6 +107,7 @@ public NoBitmapSelectorDimFilter( { super(dimension, value, extractionFn); } + @Override public Filter toFilter() { diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index 299b947a441d..a1667cb4b581 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -54,6 +54,7 @@ public class LookupDimensionSpec implements DimensionSpec private final boolean retainMissingValue; @JsonProperty + @Nullable private final String replaceMissingValueWith; @JsonProperty diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index 8ff7137fb48a..2ee0a19c2d9d 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -163,7 +163,10 @@ public Object[] parametersForTestApply() public void testApply(DimensionSpec dimensionSpec, Map map) { for (Map.Entry entry : map.entrySet()) { - Assert.assertEquals(NullHandling.emptyToNullIfNeeded(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); + Assert.assertEquals( + NullHandling.emptyToNullIfNeeded(entry.getValue()), + dimensionSpec.getExtractionFn().apply(entry.getKey()) + ); } } From 2b7ebdf1066966702edaff8360e96a1dca1d1919 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Tue, 24 Apr 2018 03:30:49 +0530 Subject: [PATCH 25/38] Fix travis ci and intellij inspections --- .../druid/server/lookup/LoadingLookupTest.java | 17 ++++++++++++++--- .../query/filter/DruidDoublePredicate.java | 2 +- .../druid/query/filter/DruidFloatPredicate.java | 3 ++- .../java/io/druid/query/filter/InDimFilter.java | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java index 66e864ed6250..ce17b2299fa5 100644 --- a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java +++ b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.server.lookup.cache.loading.LoadingCache; import org.easymock.EasyMock; import org.junit.Assert; @@ -43,13 +44,20 @@ public class LoadingLookupTest public void testApplyEmptyOrNull() { Assert.assertEquals(null, loadingLookup.apply(null)); - Assert.assertEquals(null, loadingLookup.apply("")); + if (!NullHandling.replaceWithDefault()) { + // empty string should also have same behavior + Assert.assertEquals(null, loadingLookup.apply("")); + } } @Test public void testUnapplyNull() { - Assert.assertEquals(Collections.EMPTY_LIST, loadingLookup.unapply(null)); + if (NullHandling.sqlCompatible()) { + Assert.assertEquals(Collections.EMPTY_LIST, loadingLookup.unapply(null)); + } else { + Assert.assertEquals(null, loadingLookup.unapply(null)); + } } @Test @@ -68,7 +76,10 @@ public void testUnapplyAll() throws ExecutionException .andReturn(Lists.newArrayList("key")) .once(); EasyMock.replay(reverseLookupCache); - Assert.assertEquals(ImmutableMap.of("value", Lists.newArrayList("key")), loadingLookup.unapplyAll(ImmutableSet.of("value"))); + Assert.assertEquals( + ImmutableMap.of("value", Lists.newArrayList("key")), + loadingLookup.unapplyAll(ImmutableSet.of("value")) + ); EasyMock.verify(reverseLookupCache); } diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index f1e2874ea6ee..aad43cd4edd4 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -29,7 +29,7 @@ public interface DruidDoublePredicate DruidDoublePredicate MATCH_NULL_ONLY = new DruidDoublePredicate() { @Override - public boolean applyDouble(double input) + public boolean applyDouble(@SuppressWarnings("unused") double input) { return false; } diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index 63de55c2f9f0..7510ce41d34d 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -29,7 +29,7 @@ public interface DruidFloatPredicate DruidFloatPredicate MATCH_NULL_ONLY = new DruidFloatPredicate() { @Override - public boolean applyFloat(float input) + public boolean applyFloat(@SuppressWarnings("unused") float input) { return false; } @@ -41,6 +41,7 @@ public boolean applyNull() } }; + boolean applyFloat(float input); default boolean applyNull() diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index e1ac4e4cff9d..f0c027da396f 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -60,7 +60,7 @@ public class InDimFilter implements DimFilter // Hashing threshold is not applied to String for now, String still uses ImmutableSortedSet public static final int NUMERIC_HASHING_THRESHOLD = 16; - // values can contain `null` string. + // Values can contain `null` object private final SortedSet values; private final String dimension; private final ExtractionFn extractionFn; From e88db45f56a36b14bd3f1e0009299df262b71adc Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 24 Apr 2018 22:11:42 +0530 Subject: [PATCH 26/38] handle intellij code inspection failure --- .../main/java/io/druid/query/filter/DruidDoublePredicate.java | 4 ++-- .../main/java/io/druid/query/filter/DruidFloatPredicate.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index aad43cd4edd4..08c1c601e4f4 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -19,7 +19,7 @@ package io.druid.query.filter; - +@SuppressWarnings("unused") public interface DruidDoublePredicate { DruidDoublePredicate ALWAYS_FALSE = input -> false; @@ -29,7 +29,7 @@ public interface DruidDoublePredicate DruidDoublePredicate MATCH_NULL_ONLY = new DruidDoublePredicate() { @Override - public boolean applyDouble(@SuppressWarnings("unused") double input) + public boolean applyDouble(double input) { return false; } diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index 7510ce41d34d..0eb928371248 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -22,6 +22,7 @@ /** * FloatPredicate is only supported in Java 8+, so use this to avoid boxing when a float predicate is needed. */ +@SuppressWarnings("unused") public interface DruidFloatPredicate { DruidFloatPredicate ALWAYS_FALSE = input -> false; @@ -29,7 +30,7 @@ public interface DruidFloatPredicate DruidFloatPredicate MATCH_NULL_ONLY = new DruidFloatPredicate() { @Override - public boolean applyFloat(@SuppressWarnings("unused") float input) + public boolean applyFloat(float input) { return false; } From 86270e492dbe3a47e9585dd66375bd0f34951bf1 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 8 May 2018 23:52:07 +0530 Subject: [PATCH 27/38] Fix backwards compatibility when parsing origin from empty string for PeriodGranularity --- .../main/java/io/druid/query/expression/ExprUtils.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/expression/ExprUtils.java b/processing/src/main/java/io/druid/query/expression/ExprUtils.java index 38ac85f415e0..dd0a166e5e81 100644 --- a/processing/src/main/java/io/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/io/druid/query/expression/ExprUtils.java @@ -23,6 +23,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.math.expr.Expr; +import org.apache.commons.lang.StringUtils; import org.joda.time.Chronology; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -73,7 +74,14 @@ public static PeriodGranularity toPeriodGranularity( } else { Chronology chronology = timeZone == null ? ISOChronology.getInstanceUTC() : ISOChronology.getInstance(timeZone); final Object value = originArg.eval(bindings).value(); - origin = value != null ? new DateTime(value, chronology) : null; + if (value instanceof String && StringUtils.isEmpty((String) value)) { + // We get a blank string here, when sql compatible null handling is enabled + // and expression contains empty string for for origin + // e.g timestamp_floor(\"__time\",'PT1M','','UTC') + origin = null; + } else { + origin = value != null ? new DateTime(value, chronology) : null; + } } return new PeriodGranularity(period, origin, timeZone); From 0281185eb060cefafa37f8bdcb8171238439cbdd Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Thu, 10 May 2018 02:13:34 +0530 Subject: [PATCH 28/38] review comments and fix test failures from master merge --- ...penderatorDriverRealtimeIndexTaskTest.java | 4 +- ...bleValueMatcherColumnSelectorStrategy.java | 15 +-- .../query/filter/DruidDoublePredicate.java | 4 + .../query/filter/DruidFloatPredicate.java | 1 + ...oatValueMatcherColumnSelectorStrategy.java | 15 +-- ...ongValueMatcherColumnSelectorStrategy.java | 15 +-- .../io/druid/query/filter/ValueMatcher.java | 21 +++ .../epinephelinae/RowBasedGrouperHelper.java | 2 +- .../extraction/MapLookupExtractorTest.java | 19 ++- .../timeseries/TimeseriesQueryRunnerTest.java | 6 +- .../io/druid/segment/IndexMergerTestBase.java | 127 +++++++++++++----- .../druid/segment/SchemalessTestFullTest.java | 14 +- .../segment/SchemalessTestSimpleTest.java | 3 +- .../druid/sql/calcite/CalciteQueryTest.java | 4 +- 14 files changed, 150 insertions(+), 100 deletions(-) diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index 3ca23fac1cec..f4a4aab7c4f1 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -131,6 +131,7 @@ import io.druid.timeline.DataSegment; import io.druid.timeline.partition.LinearShardSpec; import io.druid.timeline.partition.NumberedShardSpec; +import io.druid.utils.Runnables; import org.apache.commons.io.FileUtils; import org.easymock.EasyMock; import org.joda.time.DateTime; @@ -223,8 +224,7 @@ public InputRow nextRow() @Override public Runnable commit() { - return () -> { - }; + return Runnables.getNoopRunnable(); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java index a99d8a52de4f..7ff0aa2d352e 100644 --- a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java @@ -32,20 +32,7 @@ public ValueMatcher makeValueMatcher(final BaseDoubleColumnValueSelector selecto { final Double matchVal = DimensionHandlerUtils.convertObjectToDouble(value); if (matchVal == null) { - return new ValueMatcher() - { - @Override - public boolean matches() - { - return selector.isNull(); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + return ValueMatcher.nullValueMatcher(selector); } final long matchValLongBits = Double.doubleToLongBits(matchVal); diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index 08c1c601e4f4..32d788291282 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -19,6 +19,10 @@ package io.druid.query.filter; +/** + * DoublePredicate is only supported in Java 8+, so use this to avoid boxing when a double predicate is needed. + */ +// All implementations are currently lambda expressions and intellij inspections wrongly complains about unused variable. @SuppressWarnings("unused") public interface DruidDoublePredicate { diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index 0eb928371248..32173b308656 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -22,6 +22,7 @@ /** * FloatPredicate is only supported in Java 8+, so use this to avoid boxing when a float predicate is needed. */ +// All implementations are currently lambda expressions and intellij inspections wrongly complains about unused variable. @SuppressWarnings("unused") public interface DruidFloatPredicate { diff --git a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java index 5628e09bf92d..55ae117ece3b 100644 --- a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java @@ -31,20 +31,7 @@ public ValueMatcher makeValueMatcher(final BaseFloatColumnValueSelector selector { final Float matchVal = DimensionHandlerUtils.convertObjectToFloat(value); if (matchVal == null) { - return new ValueMatcher() - { - @Override - public boolean matches() - { - return selector.isNull(); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + return ValueMatcher.nullValueMatcher(selector); } final int matchValIntBits = Float.floatToIntBits(matchVal); diff --git a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java index 9b7966d7dd8f..c810eac208e4 100644 --- a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java @@ -31,20 +31,7 @@ public ValueMatcher makeValueMatcher(final BaseLongColumnValueSelector selector, { final Long matchVal = DimensionHandlerUtils.convertObjectToLong(value); if (matchVal == null) { - return new ValueMatcher() - { - @Override - public boolean matches() - { - return selector.isNull(); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + return ValueMatcher.nullValueMatcher(selector); } final long matchValLong = matchVal; return new ValueMatcher() diff --git a/processing/src/main/java/io/druid/query/filter/ValueMatcher.java b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java index cad255474d8b..955e01b0dded 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueMatcher.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java @@ -21,6 +21,8 @@ import io.druid.query.monomorphicprocessing.CalledFromHotLoop; import io.druid.query.monomorphicprocessing.HotLoopCallee; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.BaseNullableColumnValueSelector; /** */ @@ -28,4 +30,23 @@ public interface ValueMatcher extends HotLoopCallee { @CalledFromHotLoop boolean matches(); + + // Utility method to match null values. + static ValueMatcher nullValueMatcher(BaseNullableColumnValueSelector selector) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; + } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 26a7f7a568ca..691f4d3dbde2 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -923,7 +923,7 @@ private static int compareDimsInRowsWithAggs( } } - static long estimateStringKeySize(String key) + static long estimateStringKeySize(@Nullable String key) { long length = key == null ? 0 : key.length(); return length * Character.BYTES + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; diff --git a/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java b/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java index 679fff86020d..64ecad5e85c6 100644 --- a/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MapLookupExtractorTest.java @@ -65,11 +65,19 @@ public void testInjectiveUnApply() ); Assert.assertEquals(Arrays.asList("foo"), injectiveFn.unapply("bar")); - Assert.assertEquals( - "Null value should be equal to empty string", - Sets.newHashSet("null"), - Sets.newHashSet(injectiveFn.unapply((String) null)) - ); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + "Null value should be equal to empty string", + Sets.newHashSet("null"), + Sets.newHashSet(injectiveFn.unapply((String) null)) + ); + } else { + Assert.assertEquals( + "Null value should be equal to empty string", + Collections.emptySet(), + Sets.newHashSet(injectiveFn.unapply((String) null)) + ); + } Assert.assertEquals(Sets.newHashSet(""), Sets.newHashSet(injectiveFn.unapply("empty_string"))); Assert.assertEquals("not existing value returns empty list", Collections.EMPTY_LIST, injectiveFn.unapply("not There")); } @@ -83,7 +91,6 @@ public void testGetMap() @Test public void testApply() { - Assert.assertEquals("bar", fn.apply("foo")); } diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index 9549a3416143..3c2f76a62c7d 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -546,13 +546,13 @@ public void testTimeseriesIntervalOutOfRanges() new Result<>( QueryRunnerTestHelper.emptyInterval.getIntervals().get(0).getStart(), new TimeseriesResultValue( - ImmutableMap.of( + TestHelper.createExpectedMap( "rows", 0L, "index", - 0L, + NullHandling.defaultLongValue(), QueryRunnerTestHelper.addRowsIndexConstantMetric, - 1.0 + NullHandling.sqlCompatible() ? null : 1.0 ) ) ) diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 002654464c29..9aada382daeb 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -1087,43 +1087,73 @@ public void testJointDimMerge() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List rowList = RowIteratorHelper.toList(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } Assert.assertEquals(4, rowList.size()); - Assert.assertEquals( - Arrays.asList(null, "310", null, null, null, null, "910"), - rowList.get(0).dimensionValues() - ); - Assert.assertEquals( - Arrays.asList("210", "311", null, null, "710", "810", "911"), - rowList.get(1).dimensionValues() - ); - Assert.assertEquals( - Arrays.asList(null, null, "520", "620", "720", "820", "920"), - rowList.get(2).dimensionValues() - ); - Assert.assertEquals( - Arrays.asList(null, null, null, "621", null, "821", "921"), - rowList.get(3).dimensionValues() - ); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + Arrays.asList(null, "310", null, null, null, null, "910"), + rowList.get(0).dimensionValues() + ); + Assert.assertEquals( + Arrays.asList("210", "311", null, null, "710", "810", "911"), + rowList.get(1).dimensionValues() + ); + Assert.assertEquals( + Arrays.asList(null, null, "520", "620", "720", "820", "920"), + rowList.get(2).dimensionValues() + ); + Assert.assertEquals( + Arrays.asList(null, null, null, "621", null, "821", "921"), + rowList.get(3).dimensionValues() + ); + + checkBitmapIndex(Arrays.asList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Arrays.asList(0, 1, 3), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Arrays.asList(0, 3), adapter.getBitmapIndex("d7", null)); + } else { + Assert.assertEquals( + Arrays.asList("", "", "310", null, null, "", null, "910"), + rowList.get(0).dimensionValues() + ); + Assert.assertEquals( + Arrays.asList(null, "210", "311", null, null, "710", "810", "911"), + rowList.get(1).dimensionValues() + ); + Assert.assertEquals( + Arrays.asList(null, null, null, "520", "620", "720", "820", "920"), + rowList.get(2).dimensionValues() + ); + Assert.assertEquals( + Arrays.asList(null, null, null, "", "621", "", "821", "921"), + rowList.get(3).dimensionValues() + ); + checkBitmapIndex(Arrays.asList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Collections.emptyList(), adapter.getBitmapIndex("d7", null)); + } - checkBitmapIndex(Arrays.asList(0, 2, 3), adapter.getBitmapIndex("d2", null)); checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d2", "210")); checkBitmapIndex(Arrays.asList(2, 3), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Collections.singletonList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Arrays.asList(0, 1, 3), adapter.getBitmapIndex("d5", null)); checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d5", "520")); checkBitmapIndex(Arrays.asList(0, 1), adapter.getBitmapIndex("d6", null)); checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Arrays.asList(0, 3), adapter.getBitmapIndex("d7", null)); checkBitmapIndex(Collections.singletonList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Collections.singletonList(2), adapter.getBitmapIndex("d7", "720")); @@ -1230,15 +1260,33 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List rowList = RowIteratorHelper.toList(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d3", "d6", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } + Assert.assertEquals(4, rowList.size()); - Assert.assertEquals(Arrays.asList("310", null, null, "910"), rowList.get(0).dimensionValues()); - Assert.assertEquals(Arrays.asList("310", null, null, "910"), rowList.get(1).dimensionValues()); - Assert.assertEquals(Arrays.asList("310", null, null, "910"), rowList.get(2).dimensionValues()); - Assert.assertEquals(Arrays.asList(null, "621", "821", "921"), rowList.get(3).dimensionValues()); + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(Arrays.asList("310", null, null, "910"), rowList.get(0).dimensionValues()); + Assert.assertEquals(Arrays.asList("310", null, null, "910"), rowList.get(1).dimensionValues()); + Assert.assertEquals(Arrays.asList("310", null, null, "910"), rowList.get(2).dimensionValues()); + Assert.assertEquals(Arrays.asList(null, "621", "821", "921"), rowList.get(3).dimensionValues()); + } else { + Assert.assertEquals(Arrays.asList("", "", "310", null, null, "", null, "910"), rowList.get(0).dimensionValues()); + Assert.assertEquals(Arrays.asList("", "", "310", null, null, "", null, "910"), rowList.get(1).dimensionValues()); + Assert.assertEquals(Arrays.asList("", "", "310", null, null, "", null, "910"), rowList.get(2).dimensionValues()); + Assert.assertEquals( + Arrays.asList(null, null, null, "", "621", "", "821", "921"), + rowList.get(3).dimensionValues() + ); + } checkBitmapIndex(Collections.singletonList(3), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Arrays.asList(0, 1, 2), adapter.getBitmapIndex("d3", "310")); @@ -1282,7 +1330,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "1", "dimA", "") + ImmutableMap.of("dimB", "1") ) ); @@ -1290,7 +1338,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "", "dimA", "1") + ImmutableMap.of("dimA", "1") ) ); @@ -1748,7 +1796,14 @@ public void testMergeNumericDims() throws Exception Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames())); Assert.assertEquals(4, rowList.size()); - Assert.assertEquals(Arrays.asList(0L, 0.0f, "Nully Row"), rowList.get(0).dimensionValues()); + Assert.assertEquals( + Arrays.asList( + NullHandling.defaultLongValue(), + NullHandling.defaultFloatValue(), + "Nully Row" + ), + rowList.get(0).dimensionValues() + ); Assert.assertEquals(Collections.singletonList(2L), rowList.get(0).metricValues()); Assert.assertEquals(Arrays.asList(72L, 60000.789f, "World"), rowList.get(1).dimensionValues()); @@ -1830,13 +1885,13 @@ public void testPersistNullColumnSkipping() throws Exception index1.add(new MapBasedInputRow( 1L, Arrays.asList("d1", "d2"), - ImmutableMap.of("d1", "a", "d2", "", "A", 1) + ImmutableMap.of("d1", "a", "A", 1) )); index1.add(new MapBasedInputRow( 1L, Arrays.asList("d1", "d2"), - ImmutableMap.of("d1", "b", "d2", "", "A", 1) + ImmutableMap.of("d1", "b", "A", 1) )); final File tempDir = temporaryFolder.newFolder(); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java index b8a6d5ad2f3b..7f711a1564e4 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java @@ -1185,13 +1185,13 @@ public void testDifferentMetrics() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 10L) - .put("index", 900.0D) - .put("addRowsIndexConstant", 911.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", NullHandling.replaceWithDefault() ? 0.0D : 100.0D) - .build() + .put("rows", NullHandling.sqlCompatible() ? 11L : 10L) + .put("index", 900.0D) + .put("addRowsIndexConstant", NullHandling.sqlCompatible() ? 912.0D : 911.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.replaceWithDefault() ? 0.0D : 100.0D) + .build() ) ) ); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index fe2b2aa8c896..5f686c521086 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -117,7 +117,8 @@ public static Collection constructorFeeder() public SchemalessTestSimpleTest(Segment segment, boolean coalesceAbsentAndEmptyDims) { this.segment = segment; - this.coalesceAbsentAndEmptyDims = coalesceAbsentAndEmptyDims; + // Empty and empty dims are equivalent only when replaceWithDefault is true + this.coalesceAbsentAndEmptyDims = coalesceAbsentAndEmptyDims && NullHandling.replaceWithDefault(); } @Test diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 201989c5bd11..b245d8c342b7 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -303,7 +303,7 @@ public void testSelectCountStart() throws Exception .context(QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS) .build()), ImmutableList.of( - new Object[]{11.0, 0.0} + new Object[]{11.0, NullHandling.defaultDoubleValue()} ) ); @@ -326,7 +326,7 @@ public void testSelectCountStart() throws Exception .context(QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS) .build()), ImmutableList.of( - new Object[]{11.0, 0.0} + new Object[]{11.0, NullHandling.defaultDoubleValue()} ) ); From ad204a9ea3d20da7d89b566d55992b554a230cb4 Mon Sep 17 00:00:00 2001 From: Nishant Date: Thu, 10 May 2018 20:09:44 +0530 Subject: [PATCH 29/38] remove unnecessary nullToEmpty --- .../java/io/druid/query/extraction/MapLookupExtractor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index 785371fd5096..8e06aa6deb1d 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -87,9 +87,10 @@ public List unapply(@Nullable final String value) return Lists.newArrayList(Maps.filterKeys(map, new Predicate() { - @Override public boolean apply(@Nullable String key) + @Override + public boolean apply(@Nullable String key) { - return map.get(key).equals(Strings.nullToEmpty(value)); + return map.get(key).equals(value); } }).keySet()); From 0fdecc4ab633d702f3272ca0aabde5f5efab77cd Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 18 May 2018 23:23:14 +0530 Subject: [PATCH 30/38] More fixes to Druid Expressions found during testing Will Add Unit tests in subsequent PR. --- .../main/java/io/druid/data/input/Rows.java | 5 ++- .../main/java/io/druid/math/expr/Expr.java | 6 +++ .../java/io/druid/math/expr/Function.java | 5 ++- .../java/io/druid/math/expr/FunctionTest.java | 2 +- .../ApproximateHistogramAggregator.java | 5 ++- .../ApproximateHistogramAggregationTest.java | 13 +++++-- .../io/druid/indexer/InputRowSerdeTest.java | 3 +- .../expression/TimestampExtractExprMacro.java | 7 +++- .../groupby/orderby/DefaultLimitSpec.java | 2 +- .../segment/virtual/ExpressionSelectors.java | 38 ++++++++++++++++--- 10 files changed, 69 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/io/druid/data/input/Rows.java b/api/src/main/java/io/druid/data/input/Rows.java index 0ef09e9fa246..e0fbe1a172b4 100644 --- a/api/src/main/java/io/druid/data/input/Rows.java +++ b/api/src/main/java/io/druid/data/input/Rows.java @@ -23,9 +23,11 @@ import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Maps; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.parsers.ParseException; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -92,10 +94,11 @@ public static List objectToStrings(final Object inputValue) * @throws NullPointerException if the string is null * @throws ParseException if the column cannot be converted to a number */ + @Nullable public static Number objectToNumber(final String name, final Object inputValue) { if (inputValue == null) { - return Rows.LONG_ZERO; + return NullHandling.defaultLongValue(); } if (inputValue instanceof Number) { diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index ae55e7d4dcb2..b7a59f240d43 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -272,6 +272,9 @@ class UnaryMinusExpr extends UnaryExpr public ExprEval eval(ObjectBinding bindings) { ExprEval ret = expr.eval(bindings); + if (NullHandling.sqlCompatible() && (ret.value() == null)) { + return ExprEval.of(null); + } if (ret.type() == ExprType.LONG) { return ExprEval.of(-ret.asLong()); } @@ -307,6 +310,9 @@ class UnaryNotExpr extends UnaryExpr public ExprEval eval(ObjectBinding bindings) { ExprEval ret = expr.eval(bindings); + if (NullHandling.sqlCompatible() && (ret.value() == null)) { + return ExprEval.of(null); + } // conforming to other boolean-returning binary operators ExprType retType = ret.type() == ExprType.DOUBLE ? ExprType.DOUBLE : ExprType.LONG; return ExprEval.of(!ret.asBoolean(), retType); diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 4116030d6288..20a5b21b2608 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -796,6 +796,9 @@ public String name() @Override protected ExprEval eval(ExprEval x, ExprEval y) { + if (NullHandling.sqlCompatible() && x.value() == null) { + return ExprEval.of(null); + } ExprType castTo; try { castTo = ExprType.valueOf(StringUtils.toUpperCase(y.asString())); @@ -937,7 +940,7 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final String arg = args.get(0).eval(bindings).asString(); - return arg == null ? ExprEval.of(0) : ExprEval.of(arg.length()); + return arg == null ? ExprEval.ofLong(NullHandling.defaultLongValue()) : ExprEval.of(arg.length()); } } diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java index 9ea6efb2c6cb..89b997a116f3 100644 --- a/common/src/test/java/io/druid/math/expr/FunctionTest.java +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -88,7 +88,7 @@ public void testSubstring() public void testStrlen() { assertExpr("strlen(x)", 3L); - assertExpr("strlen(nonexistent)", 0L); + assertExpr("strlen(nonexistent)", NullHandling.defaultLongValue()); } @Test diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java index 6ed641d79c98..3703e38b610c 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregator.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.histogram; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.query.aggregation.Aggregator; import io.druid.segment.BaseFloatColumnValueSelector; @@ -59,7 +60,9 @@ public ApproximateHistogramAggregator( @Override public void aggregate() { - histogram.offer(selector.getFloat()); + if (NullHandling.replaceWithDefault() || !selector.isNull()) { + histogram.offer(selector.getFloat()); + } } @Override diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java index ae5904c8cc82..94a032f24405 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregationTest.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.histogram; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequence; @@ -79,10 +80,14 @@ public void testIngestWithNullsIgnoredAndQuery() throws Exception @Test public void testIngestWithNullsToZeroAndQuery() throws Exception { - MapBasedRow row = ingestAndQuery(false); - Assert.assertEquals(0.0, row.getMetric("index_min").floatValue(), 0.0001); - Assert.assertEquals(135.109191, row.getMetric("index_max").floatValue(), 0.0001); - Assert.assertEquals(131.428176, row.getMetric("index_quantile").floatValue(), 0.0001); + // Nulls are ignored and not replaced with default for SQL compatible null handling. + // This is already tested in testIngestWithNullsIgnoredAndQuery() + if (NullHandling.replaceWithDefault()) { + MapBasedRow row = ingestAndQuery(false); + Assert.assertEquals(0.0F, row.getMetric("index_min")); + Assert.assertEquals(135.109191, row.getMetric("index_max").floatValue(), 0.0001); + Assert.assertEquals(131.428176, row.getMetric("index_quantile").floatValue(), 0.0001); + } } private MapBasedRow ingestAndQuery(boolean ignoreNulls) throws Exception diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index 18efcdfd1938..41ac18984238 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionsSpec; @@ -152,7 +153,7 @@ public void testSerde() Assert.assertEquals(300.1f, out.getRaw("d4")); Assert.assertEquals(400.5d, out.getRaw("d5")); - Assert.assertEquals(0.0f, out.getMetric("agg_non_existing").floatValue(), 0.00001); + Assert.assertEquals(NullHandling.defaultFloatValue(), out.getMetric("agg_non_existing")); Assert.assertEquals(5.0f, out.getMetric("m1out").floatValue(), 0.00001); Assert.assertEquals(100L, out.getMetric("m2out")); Assert.assertEquals(1, ((HyperLogLogCollector) out.getRaw("m3out")).estimateCardinality(), 0.001); diff --git a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java index bb1b2ca5f243..bdea9147c0f8 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java @@ -88,7 +88,12 @@ class TimestampExtractExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - final DateTime dateTime = new DateTime(arg.eval(bindings).asLong(), chronology); + Object val = arg.eval(bindings).value(); + if (val == null) { + // Return null if the argument if null. + return ExprEval.of(null); + } + final DateTime dateTime = new DateTime(val, chronology); switch (unit) { case EPOCH: return ExprEval.of(dateTime.getMillis() / 1000); diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index 2cbaf81bfe99..557d71e1f97f 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -269,7 +269,7 @@ public int compare(Row left, Row right) private Ordering metricOrdering(final String column, final Comparator comparator) { - return Ordering.from(Comparator.comparing((Row row) -> row.getRaw(column), Comparator.nullsLast(comparator))); + return Ordering.from(Comparator.comparing((Row row) -> row.getRaw(column), Comparator.nullsFirst(comparator))); } private Ordering dimensionOrdering(final String dimension, final StringComparator comparator) diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index fe67f65adad2..c3f16572c592 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -248,23 +248,32 @@ private static Expr.ObjectBinding createBindings(Expr expression, ColumnSelector { final Map> suppliers = Maps.newHashMap(); for (String columnName : Parser.findRequiredBindings(expression)) { - final ColumnCapabilities columnCapabilities = columnSelectorFactory.getColumnCapabilities(columnName); + final ColumnCapabilities columnCapabilities = columnSelectorFactory + .getColumnCapabilities(columnName); final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; final Supplier supplier; if (nativeType == ValueType.FLOAT) { - supplier = columnSelectorFactory.makeColumnValueSelector(columnName)::getFloat; + ColumnValueSelector selector = columnSelectorFactory + .makeColumnValueSelector(columnName); + supplier = makeNullableSupplier(selector, selector::getFloat); } else if (nativeType == ValueType.LONG) { - supplier = columnSelectorFactory.makeColumnValueSelector(columnName)::getLong; + ColumnValueSelector selector = columnSelectorFactory + .makeColumnValueSelector(columnName); + supplier = makeNullableSupplier(selector, selector::getLong); } else if (nativeType == ValueType.DOUBLE) { - supplier = columnSelectorFactory.makeColumnValueSelector(columnName)::getDouble; + ColumnValueSelector selector = columnSelectorFactory + .makeColumnValueSelector(columnName); + supplier = makeNullableSupplier(selector, selector::getDouble); } else if (nativeType == ValueType.STRING) { supplier = supplierFromDimensionSelector( - columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName)) + columnSelectorFactory + .makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName)) ); } else if (nativeType == null) { // Unknown ValueType. Try making an Object selector and see if that gives us anything useful. - supplier = supplierFromObjectSelector(columnSelectorFactory.makeColumnValueSelector(columnName)); + supplier = supplierFromObjectSelector(columnSelectorFactory + .makeColumnValueSelector(columnName)); } else { // Unhandleable ValueType (COMPLEX). supplier = null; @@ -292,6 +301,23 @@ private static Expr.ObjectBinding createBindings(Expr expression, ColumnSelector } } + private static Supplier makeNullableSupplier( + ColumnValueSelector selector, + Supplier supplier + ) + { + if (NullHandling.replaceWithDefault()) { + return supplier; + } else { + return () -> { + if (selector.isNull()) { + return null; + } + return supplier.get(); + }; + } + } + @VisibleForTesting @Nonnull static Supplier supplierFromDimensionSelector(final DimensionSelector selector) From 78e3b08384979bcf99f0909d7c7799367fba2c85 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 22 May 2018 19:50:22 +0530 Subject: [PATCH 31/38] review comments - Extract out null handling for GroupBySerdeHelpers --- .../io/druid/indexer/InputRowSerdeTest.java | 2 +- .../query/extraction/MapLookupExtractor.java | 10 +- .../query/filter/DruidDoublePredicate.java | 4 +- .../query/filter/DruidFloatPredicate.java | 4 +- .../query/filter/DruidLongPredicate.java | 3 +- .../epinephelinae/RowBasedGrouperHelper.java | 174 +++++++++++------- 6 files changed, 115 insertions(+), 82 deletions(-) diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index 41ac18984238..578be78d5257 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -153,7 +153,7 @@ public void testSerde() Assert.assertEquals(300.1f, out.getRaw("d4")); Assert.assertEquals(400.5d, out.getRaw("d5")); - Assert.assertEquals(NullHandling.defaultFloatValue(), out.getMetric("agg_non_existing")); + Assert.assertEquals(NullHandling.defaultDoubleValue(), out.getMetric("agg_non_existing")); Assert.assertEquals(5.0f, out.getMetric("m1out").floatValue(), 0.00001); Assert.assertEquals(100L, out.getMetric("m2out")); Assert.assertEquals(1, ((HyperLogLogCollector) out.getRaw("m3out")).estimateCardinality(), 0.001); diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index 8e06aa6deb1d..3cf7ef3516ea 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; @@ -85,14 +84,7 @@ public List unapply(@Nullable final String value) return Collections.EMPTY_LIST; } - return Lists.newArrayList(Maps.filterKeys(map, new Predicate() - { - @Override - public boolean apply(@Nullable String key) - { - return map.get(key).equals(value); - } - }).keySet()); + return Lists.newArrayList(Maps.filterKeys(map, key -> map.get(key).equals(valueToLookup)).keySet()); } diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index 32d788291282..d3ab89c36c8b 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -20,9 +20,11 @@ package io.druid.query.filter; /** - * DoublePredicate is only supported in Java 8+, so use this to avoid boxing when a double predicate is needed. + * Note: this is not a {@link io.druid.guice.annotations.PublicApi} or an + * {@link io.druid.guice.annotations.ExtensionPoint} of Druid. */ // All implementations are currently lambda expressions and intellij inspections wrongly complains about unused variable. +// SupressWarnings can be removed once https://youtrack.jetbrains.com/issue/IDEA-191743 is resolved. @SuppressWarnings("unused") public interface DruidDoublePredicate { diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index 32173b308656..77dfb306ac48 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -20,9 +20,11 @@ package io.druid.query.filter; /** - * FloatPredicate is only supported in Java 8+, so use this to avoid boxing when a float predicate is needed. + * Note: this is not a {@link io.druid.guice.annotations.PublicApi} or an + * {@link io.druid.guice.annotations.ExtensionPoint} of Druid. */ // All implementations are currently lambda expressions and intellij inspections wrongly complains about unused variable. +// SupressWarnings can be removed once https://youtrack.jetbrains.com/issue/IDEA-191743 is resolved. @SuppressWarnings("unused") public interface DruidFloatPredicate { diff --git a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java index 4175c6bcbe18..ecf15c03e04a 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java @@ -20,7 +20,8 @@ package io.druid.query.filter; /** - * LongPredicate is only supported in Java 8+, so use this to avoid boxing when a long predicate is needed. + * Note: this is not a {@link io.druid.guice.annotations.PublicApi} or an + * {@link io.druid.guice.annotations.ExtensionPoint} of Druid. */ public interface DruidLongPredicate { diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 691f4d3dbde2..21c402be4fbb 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -1198,7 +1198,7 @@ public Grouper.BufferComparator bufferComparatorWithAggregators( throw new IAE("Cannot order by a non-numeric aggregator[%s]", orderSpec); } - serdeHelper = makeNumericSerdeHelper(valueType, aggOffset, true, stringComparator); + serdeHelper = makeNullHandlingNumericserdeHelper(valueType, aggOffset, true, stringComparator); orderByHelpers.add(serdeHelper); needsReverses.add(needsReverse); @@ -1383,12 +1383,31 @@ private RowBasedKeySerdeHelper makeSerdeHelper( case LONG: case FLOAT: case DOUBLE: - return makeNumericSerdeHelper(valueType, keyBufferPosition, pushLimitDown, stringComparator); + return makeNullHandlingNumericserdeHelper(valueType, keyBufferPosition, pushLimitDown, stringComparator); default: throw new IAE("invalid type: %s", valueType); } } + private RowBasedKeySerdeHelper makeNullHandlingNumericserdeHelper( + ValueType valueType, + int keyBufferPosition, + boolean pushLimitDown, + @Nullable StringComparator stringComparator + ) + { + if (NullHandling.sqlCompatible()) { + return new NullableRowBasedKeySerdeHelper(makeNumericSerdeHelper( + valueType, + keyBufferPosition + Byte.BYTES, + pushLimitDown, + stringComparator + ), keyBufferPosition); + } else { + return makeNumericSerdeHelper(valueType, keyBufferPosition, pushLimitDown, stringComparator); + } + } + private RowBasedKeySerdeHelper makeNumericSerdeHelper( ValueType valueType, int keyBufferPosition, @@ -1549,13 +1568,13 @@ private class LongRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> Longs.compare( - lhsBuffer.getLong(lhsPosition + keyBufferPosition + Byte.BYTES), - rhsBuffer.getLong(rhsPosition + keyBufferPosition + Byte.BYTES) + lhsBuffer.getLong(lhsPosition + keyBufferPosition), + rhsBuffer.getLong(rhsPosition + keyBufferPosition) ); } else { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - long lhs = lhsBuffer.getLong(lhsPosition + keyBufferPosition + Byte.BYTES); - long rhs = rhsBuffer.getLong(rhsPosition + keyBufferPosition + Byte.BYTES); + long lhs = lhsBuffer.getLong(lhsPosition + keyBufferPosition); + long rhs = rhsBuffer.getLong(rhsPosition + keyBufferPosition); return stringComparator.compare(String.valueOf(lhs), String.valueOf(rhs)); }; @@ -1565,36 +1584,20 @@ private class LongRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Long.BYTES + Byte.BYTES; + return Long.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - Long aLong = (Long) key.getKey()[idx]; - if (aLong == null) { - keyBuffer.put((byte) 1); - keyBuffer.putLong(0L); - } else { - keyBuffer.put((byte) 0); - keyBuffer.putLong(aLong); - } + keyBuffer.putLong(DimensionHandlerUtils.nullToZero((Long) key.getKey()[idx])); return true; } - protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) - { - if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandling.defaultLongValue(); - } else { - return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); - } - } - @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); + dimValues[dimValIdx] = buffer.getLong(initialOffset + keyBufferPosition); } @Override @@ -1612,19 +1615,18 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper FloatRowBasedKeySerdeHelper( int keyBufferPosition, boolean pushLimitDown, - @Nullable StringComparator stringComparator - ) + @Nullable StringComparator stringComparator) { this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> Float.compare( - lhsBuffer.getFloat(lhsPosition + keyBufferPosition + Byte.BYTES), - rhsBuffer.getFloat(rhsPosition + keyBufferPosition + Byte.BYTES) + lhsBuffer.getFloat(lhsPosition + keyBufferPosition), + rhsBuffer.getFloat(rhsPosition + keyBufferPosition) ); } else { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - float lhs = lhsBuffer.getFloat(lhsPosition + keyBufferPosition + Byte.BYTES); - float rhs = rhsBuffer.getFloat(rhsPosition + keyBufferPosition + Byte.BYTES); + float lhs = lhsBuffer.getFloat(lhsPosition + keyBufferPosition); + float rhs = rhsBuffer.getFloat(rhsPosition + keyBufferPosition); return stringComparator.compare(String.valueOf(lhs), String.valueOf(rhs)); }; } @@ -1633,36 +1635,20 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Float.BYTES + Byte.BYTES; + return Float.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - Float aFloat = (Float) key.getKey()[idx]; - if (aFloat == null) { - keyBuffer.put((byte) 1); - keyBuffer.putFloat(0F); - } else { - keyBuffer.put((byte) 0); - keyBuffer.putFloat(aFloat); - } + keyBuffer.putFloat(DimensionHandlerUtils.nullToZero((Float) key.getKey()[idx])); return true; } - protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) - { - if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandling.defaultFloatValue(); - } else { - return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); - } - } - @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); + dimValues[dimValIdx] = buffer.getFloat(initialOffset + keyBufferPosition); } @Override @@ -1686,13 +1672,13 @@ private class DoubleRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> Double.compare( - lhsBuffer.getDouble(lhsPosition + keyBufferPosition + Byte.BYTES), - rhsBuffer.getDouble(rhsPosition + keyBufferPosition + Byte.BYTES) + lhsBuffer.getDouble(lhsPosition + keyBufferPosition), + rhsBuffer.getDouble(rhsPosition + keyBufferPosition) ); } else { bufferComparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - double lhs = lhsBuffer.getDouble(lhsPosition + keyBufferPosition + Byte.BYTES); - double rhs = rhsBuffer.getDouble(rhsPosition + keyBufferPosition + Byte.BYTES); + double lhs = lhsBuffer.getDouble(lhsPosition + keyBufferPosition); + double rhs = rhsBuffer.getDouble(rhsPosition + keyBufferPosition); return stringComparator.compare(String.valueOf(lhs), String.valueOf(rhs)); }; } @@ -1701,42 +1687,92 @@ private class DoubleRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Double.BYTES + Byte.BYTES; + return Double.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - Double aDouble = (Double) key.getKey()[idx]; - if (aDouble == null) { - keyBuffer.put((byte) 1); - keyBuffer.putDouble(0.0D); - } else { - keyBuffer.put((byte) 0); - keyBuffer.putDouble(aDouble); - } + keyBuffer.putDouble(DimensionHandlerUtils.nullToZero((Double) key.getKey()[idx])); return true; } - protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) + @Override + public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) + { + dimValues[dimValIdx] = buffer.getDouble(initialOffset + keyBufferPosition); + } + + @Override + public BufferComparator getBufferComparator() { - if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { - return NullHandling.defaultDoubleValue(); + return bufferComparator; + } + } + + private class NullableRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper + { + public static final byte IS_NULL_BYTE = (byte) 1; + public static final byte IS_NOT_NULL_BYTE = (byte) 1; + private final RowBasedKeySerdeHelper delegate; + private final int keyBufferPosition; + private final BufferComparator comparator; + + NullableRowBasedKeySerdeHelper(RowBasedKeySerdeHelper delegate, int keyBufferPosition) + { + this.delegate = delegate; + this.keyBufferPosition = keyBufferPosition; + BufferComparator delegateBufferComparator = this.delegate.getBufferComparator(); + this.comparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { + boolean isLhsNull = lhsBuffer.get(lhsPosition + keyBufferPosition) == IS_NULL_BYTE; + boolean isrhsNull = rhsBuffer.get(rhsPosition + keyBufferPosition) == IS_NULL_BYTE; + if (isLhsNull && isrhsNull) { + // Both are null + return 0; + } + if (isLhsNull) { + return -1; + } + if (isrhsNull) { + return 1; + } + return delegateBufferComparator.compare(lhsBuffer, rhsBuffer, lhsPosition, rhsPosition); + }; + } + + @Override + public int getKeyBufferValueSize() + { + return delegate.getKeyBufferValueSize() + Byte.BYTES; + } + + @Override + public boolean putToKeyBuffer(RowBasedKey key, int idx) + { + Object val = key.getKey()[idx]; + if (val == null) { + keyBuffer.put(IS_NULL_BYTE); } else { - return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); + keyBuffer.put((byte) 0); } + delegate.putToKeyBuffer(key, idx); + return true; } @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + dimValues[dimValIdx] = null; + } else { + delegate.getFromByteBuffer(buffer, initialOffset, dimValIdx, dimValues); + } } @Override public BufferComparator getBufferComparator() { - return bufferComparator; + return comparator; } } } From a9e9179a1ef467c48ef65b9dd98c31ceea13a9b9 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Wed, 23 May 2018 03:30:55 +0530 Subject: [PATCH 32/38] Extract NullableGroupBycolumnSelectorStrategy, Null Bytes fix failing UTs * Extract NullableGroupBycolumnSelectorStrategy - only used when null handling is enabled, fix failing UTs * fix test after making extracting groupby changes * Set AWS region in travis --- .travis.yml | 3 +- .../indexing/kafka/KafkaIndexTaskTest.java | 19 ++-- .../java/io/druid/indexer/InputRowSerde.java | 9 +- .../io/druid/common/config/NullHandling.java | 2 + .../aggregation/NullableBufferAggregator.java | 13 +-- .../io/druid/query/filter/InDimFilter.java | 2 +- .../druid/query/filter/SelectorDimFilter.java | 2 +- .../epinephelinae/GroupByQueryEngineV2.java | 19 +++- .../epinephelinae/RowBasedGrouperHelper.java | 23 ++-- ...ngStringGroupByColumnSelectorStrategy.java | 9 +- .../DoubleGroupByColumnSelectorStrategy.java | 32 ++---- .../FloatGroupByColumnSelectorStrategy.java | 32 ++---- .../column/GroupByColumnSelectorStrategy.java | 4 +- .../LongGroupByColumnSelectorStrategy.java | 34 ++---- ...bleValueGroupByColumnSelectorStrategy.java | 106 ++++++++++++++++++ .../StringGroupByColumnSelectorStrategy.java | 9 +- .../appenderator/AppenderatorTest.java | 54 ++++++--- 17 files changed, 238 insertions(+), 134 deletions(-) create mode 100644 processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java diff --git a/.travis.yml b/.travis.yml index 1bfa5ed7141f..db4f10ecbc79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ matrix: before_script: - unset _JAVA_OPTIONS # Server module test is run without the parallel-test option because it's memory sensitive and often fails with that option. - script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -pl server + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -pl server -Ddruid.generic.useDefaultValueForNull=false # other modules test @@ -77,6 +77,7 @@ matrix: - sudo: false env: - NAME="other modules test with SQL Compatibility" + - AWS_REGION=us-east-1 # set a aws region for unit tests install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index e3b15b0c1950..59f396b49e59 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -36,26 +36,25 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import io.druid.data.input.impl.FloatDimensionSchema; -import io.druid.data.input.impl.LongDimensionSchema; -import io.druid.data.input.impl.StringDimensionSchema; -import io.druid.indexer.TaskMetricsUtils; -import io.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; -import io.druid.indexing.common.TaskReport; -import io.druid.indexing.common.TaskReportFileWriter; -import io.druid.indexing.common.task.IndexTaskTest; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.FloatDimensionSchema; import io.druid.data.input.impl.JSONParseSpec; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.data.input.impl.StringInputRowParser; import io.druid.data.input.impl.TimestampSpec; import io.druid.discovery.DataNodeService; import io.druid.discovery.DruidNodeAnnouncer; import io.druid.discovery.LookupNodeService; +import io.druid.indexer.TaskMetricsUtils; import io.druid.indexer.TaskState; +import io.druid.indexing.common.IngestionStatsAndErrorsTaskReportData; import io.druid.indexing.common.SegmentLoaderFactory; import io.druid.indexing.common.TaskLock; +import io.druid.indexing.common.TaskReport; +import io.druid.indexing.common.TaskReportFileWriter; import io.druid.indexing.common.TaskStatus; import io.druid.indexing.common.TaskToolbox; import io.druid.indexing.common.TaskToolboxFactory; @@ -65,6 +64,7 @@ import io.druid.indexing.common.actions.TaskActionToolbox; import io.druid.indexing.common.config.TaskConfig; import io.druid.indexing.common.config.TaskStorageConfig; +import io.druid.indexing.common.task.IndexTaskTest; import io.druid.indexing.common.task.Task; import io.druid.indexing.kafka.supervisor.KafkaSupervisor; import io.druid.indexing.kafka.test.TestBroker; @@ -116,6 +116,7 @@ import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.query.timeseries.TimeseriesResultValue; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.IndexIO; import io.druid.segment.QueryableIndex; import io.druid.segment.TestHelper; @@ -2325,7 +2326,7 @@ public long countEvents(final Task task) List> results = task.getQueryRunner(query).run(wrap(query), ImmutableMap.of()).toList(); - return results.isEmpty() ? 0 : results.get(0).getValue().getLongMetric("rows"); + return results.isEmpty() ? 0L : DimensionHandlerUtils.nullToZero(results.get(0).getValue().getLongMetric("rows")); } private static byte[] JB(String timestamp, String dim1, String dim2, String dimLong, String dimFloat, String met1) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 98e141c16a03..3c7aa188691c 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -26,6 +26,7 @@ import com.google.common.io.ByteArrayDataOutput; import com.google.common.io.ByteStreams; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.Rows; @@ -57,8 +58,6 @@ public class InputRowSerde { private static final Logger log = new Logger(InputRowSerde.class); - private static final byte NULL_BYTE = (byte) 1; - private static final byte NON_NULL_BYTE = (byte) 0; private static final IndexSerdeTypeHelper STRING_HELPER = new StringIndexSerdeTypeHelper(); private static final IndexSerdeTypeHelper LONG_HELPER = new LongIndexSerdeTypeHelper(); @@ -334,9 +333,9 @@ public static final SerializeResult toBytes( String t = aggFactory.getTypeName(); if (agg.isNull()) { - out.writeByte(NULL_BYTE); + out.writeByte(NullHandling.IS_NULL_BYTE); } else { - out.writeByte(NON_NULL_BYTE); + out.writeByte(NullHandling.IS_NOT_NULL_BYTE); if (t.equals("float")) { out.writeFloat(agg.getFloat()); } else if (t.equals("long")) { @@ -461,7 +460,7 @@ public static final InputRow fromBytes( String metric = readString(in); String type = getType(metric, aggs, i); byte metricNullability = in.readByte(); - if (metricNullability == NULL_BYTE) { + if (metricNullability == NullHandling.IS_NULL_BYTE) { // metric value is null. continue; } diff --git a/java-util/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java index c1ee4dd3ab44..ca2ddd1e19e7 100644 --- a/java-util/src/main/java/io/druid/common/config/NullHandling.java +++ b/java-util/src/main/java/io/druid/common/config/NullHandling.java @@ -41,6 +41,8 @@ public class NullHandling public static final Double ZERO_DOUBLE = 0.0d; public static final Float ZERO_FLOAT = 0.0f; public static final Long ZERO_LONG = 0L; + public static final byte IS_NULL_BYTE = (byte) 1; + public static final byte IS_NOT_NULL_BYTE = (byte) 0; /** * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index a462328912bb..9695301bd0ac 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation; +import io.druid.common.config.NullHandling; import io.druid.segment.BaseNullableColumnValueSelector; import javax.annotation.Nullable; @@ -32,8 +33,6 @@ */ public class NullableBufferAggregator implements BufferAggregator { - private static final byte IS_NULL_BYTE = (byte) 0; - private static final byte IS_NOT_NULL_BYTE = (byte) 1; private final BufferAggregator delegate; private final BaseNullableColumnValueSelector selector; @@ -47,7 +46,7 @@ public NullableBufferAggregator(BufferAggregator delegate, BaseNullableColumnVal @Override public void init(ByteBuffer buf, int position) { - buf.put(position, IS_NULL_BYTE); + buf.put(position, NullHandling.IS_NULL_BYTE); delegate.init(buf, position + Byte.BYTES); } @@ -56,8 +55,8 @@ public void aggregate(ByteBuffer buf, int position) { boolean isNotNull = !selector.isNull(); if (isNotNull) { - if (buf.get(position) == IS_NULL_BYTE) { - buf.put(position, IS_NOT_NULL_BYTE); + if (buf.get(position) == NullHandling.IS_NULL_BYTE) { + buf.put(position, NullHandling.IS_NOT_NULL_BYTE); } delegate.aggregate(buf, position + Byte.BYTES); } @@ -67,7 +66,7 @@ public void aggregate(ByteBuffer buf, int position) @Nullable public Object get(ByteBuffer buf, int position) { - if (buf.get(position) == IS_NULL_BYTE) { + if (buf.get(position) == NullHandling.IS_NULL_BYTE) { return null; } return delegate.get(buf, position + Byte.BYTES); @@ -103,7 +102,7 @@ public double getDouble(ByteBuffer buf, int position) @Override public boolean isNull(ByteBuffer buf, int position) { - return buf.get(position) == IS_NULL_BYTE || delegate.isNull(buf, position + Byte.BYTES); + return buf.get(position) == NullHandling.IS_NULL_BYTE || delegate.isNull(buf, position + Byte.BYTES); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index f0c027da396f..7f4ad25e8a47 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -137,7 +137,7 @@ public byte[] getCacheKey() .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) .put(DimFilterUtils.STRING_SEPARATOR) - .put(hasNull ? (byte) 1 : (byte) 0) + .put(hasNull ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE) .put(DimFilterUtils.STRING_SEPARATOR); for (byte[] bytes : valuesBytes) { filterCacheKey.put(bytes) diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 060909324f1f..57ceca482284 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -83,7 +83,7 @@ public byte[] getCacheKey() .put(DimFilterUtils.STRING_SEPARATOR) .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) - .put(value == null ? (byte) 1 : (byte) 0) + .put(value == null ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE) .put(valueBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 9d7f81ed7122..f9063b552587 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -45,6 +45,7 @@ import io.druid.query.groupby.epinephelinae.column.GroupByColumnSelectorPlus; import io.druid.query.groupby.epinephelinae.column.GroupByColumnSelectorStrategy; import io.druid.query.groupby.epinephelinae.column.LongGroupByColumnSelectorStrategy; +import io.druid.query.groupby.epinephelinae.column.NullableValueGroupByColumnSelectorStrategy; import io.druid.query.groupby.epinephelinae.column.StringGroupByColumnSelectorStrategy; import io.druid.query.groupby.strategy.GroupByStrategyV2; import io.druid.segment.ColumnValueSelector; @@ -250,15 +251,24 @@ public GroupByColumnSelectorStrategy makeColumnSelectorStrategy( return new DictionaryBuildingStringGroupByColumnSelectorStrategy(); } case LONG: - return new LongGroupByColumnSelectorStrategy(); + return makeNullableStrategy(new LongGroupByColumnSelectorStrategy()); case FLOAT: - return new FloatGroupByColumnSelectorStrategy(); + return makeNullableStrategy(new FloatGroupByColumnSelectorStrategy()); case DOUBLE: - return new DoubleGroupByColumnSelectorStrategy(); + return makeNullableStrategy(new DoubleGroupByColumnSelectorStrategy()); default: throw new IAE("Cannot create query type helper from invalid type [%s]", type); } } + + private GroupByColumnSelectorStrategy makeNullableStrategy(GroupByColumnSelectorStrategy delegate) + { + if (NullHandling.sqlCompatible()) { + return new NullableValueGroupByColumnSelectorStrategy(delegate); + } else { + return delegate; + } + } } private abstract static class GroupByEngineIterator implements Iterator, Closeable @@ -546,7 +556,8 @@ protected void putToMap(ByteBuffer key, Map map) selectorPlus.getColumnSelectorStrategy().processValueFromGroupingKey( selectorPlus, key, - map + map, + selectorPlus.getKeyBufferPosition() ); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 21c402be4fbb..df0275acd171 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -29,8 +29,8 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.ListeningExecutorService; -import io.druid.common.config.NullHandling; import io.druid.collections.ReferenceCountingResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.common.utils.IntArrayUtils; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; @@ -1712,8 +1712,6 @@ public BufferComparator getBufferComparator() private class NullableRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper { - public static final byte IS_NULL_BYTE = (byte) 1; - public static final byte IS_NOT_NULL_BYTE = (byte) 1; private final RowBasedKeySerdeHelper delegate; private final int keyBufferPosition; private final BufferComparator comparator; @@ -1724,19 +1722,26 @@ private class NullableRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper this.keyBufferPosition = keyBufferPosition; BufferComparator delegateBufferComparator = this.delegate.getBufferComparator(); this.comparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { - boolean isLhsNull = lhsBuffer.get(lhsPosition + keyBufferPosition) == IS_NULL_BYTE; - boolean isrhsNull = rhsBuffer.get(rhsPosition + keyBufferPosition) == IS_NULL_BYTE; + boolean isLhsNull = (lhsBuffer.get(lhsPosition + keyBufferPosition) == NullHandling.IS_NULL_BYTE); + boolean isrhsNull = (rhsBuffer.get(rhsPosition + keyBufferPosition) == NullHandling.IS_NULL_BYTE); if (isLhsNull && isrhsNull) { // Both are null return 0; } + // only lhs is null if (isLhsNull) { return -1; } + // only rhs is null if (isrhsNull) { return 1; } - return delegateBufferComparator.compare(lhsBuffer, rhsBuffer, lhsPosition, rhsPosition); + return delegateBufferComparator.compare( + lhsBuffer, + rhsBuffer, + lhsPosition, + rhsPosition + ); }; } @@ -1751,9 +1756,9 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) { Object val = key.getKey()[idx]; if (val == null) { - keyBuffer.put(IS_NULL_BYTE); + keyBuffer.put(NullHandling.IS_NULL_BYTE); } else { - keyBuffer.put((byte) 0); + keyBuffer.put(NullHandling.IS_NOT_NULL_BYTE); } delegate.putToKeyBuffer(key, idx); return true; @@ -1762,7 +1767,7 @@ public boolean putToKeyBuffer(RowBasedKey key, int idx) @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + if (buffer.get(initialOffset + keyBufferPosition) == NullHandling.IS_NULL_BYTE) { dimValues[dimValIdx] = null; } else { delegate.getFromByteBuffer(buffer, initialOffset, dimValIdx, dimValues); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index b302a49b3883..d9db08985cb2 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -48,9 +48,14 @@ public class DictionaryBuildingStringGroupByColumnSelectorStrategy extends Strin } @Override - public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap) + public void processValueFromGroupingKey( + GroupByColumnSelectorPlus selectorPlus, + ByteBuffer key, + Map resultMap, + int keyBufferPosition + ) { - final int id = key.getInt(selectorPlus.getKeyBufferPosition()); + final int id = key.getInt(keyBufferPosition); // GROUP_BY_MISSING_VALUE is used to indicate empty rows, which are omitted from the result map. if (id != GROUP_BY_MISSING_VALUE) { diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java index dff778a0e19b..972d519bec02 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java @@ -20,8 +20,8 @@ package io.druid.query.groupby.epinephelinae.column; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; +import io.druid.segment.DimensionHandlerUtils; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -32,52 +32,34 @@ public class DoubleGroupByColumnSelectorStrategy implements GroupByColumnSelecto @Override public int getGroupingKeySize() { - return Double.BYTES + Byte.BYTES; + return Double.BYTES; } @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap + GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition ) { - if (key.get(selectorPlus.getKeyBufferPosition()) == (byte) 1) { - resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultDoubleValue()); - } else { - final double val = key.getDouble(selectorPlus.getKeyBufferPosition() + Byte.BYTES); - resultMap.put(selectorPlus.getOutputName(), val); - } + final double val = key.getDouble(keyBufferPosition); + resultMap.put(selectorPlus.getOutputName(), val); } @Override public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] values) { - if (NullHandling.sqlCompatible() && selector.isNull()) { - values[columnIndex] = null; - } else { - values[columnIndex] = selector.getDouble(); - } + values[columnIndex] = selector.getDouble(); } @Override - @Nullable public Object getOnlyValue(ColumnValueSelector selector) { - if (NullHandling.sqlCompatible() && selector.isNull()) { - return null; - } return selector.getDouble(); } @Override public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBuffer keyBuffer) { - if (obj == null) { - keyBuffer.put(keyBufferPosition, (byte) 1); - keyBuffer.putDouble(keyBufferPosition + Byte.BYTES, 0.0d); - } else { - keyBuffer.put(keyBufferPosition, (byte) 0); - keyBuffer.putDouble(keyBufferPosition + Byte.BYTES, (Double) obj); - } + keyBuffer.putDouble(keyBufferPosition, DimensionHandlerUtils.nullToZero((Double) obj)); } @Override diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java index 7df4ddd2f453..6a619126462f 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java @@ -19,8 +19,8 @@ package io.druid.query.groupby.epinephelinae.column; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; +import io.druid.segment.DimensionHandlerUtils; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -32,52 +32,34 @@ public class FloatGroupByColumnSelectorStrategy implements GroupByColumnSelector @Override public int getGroupingKeySize() { - return Float.BYTES + Byte.BYTES; + return Float.BYTES; } @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap + GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition ) { - if (key.get(selectorPlus.getKeyBufferPosition()) == (byte) 1) { - resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultFloatValue()); - } else { - final float val = key.getFloat(selectorPlus.getKeyBufferPosition() + Byte.BYTES); - resultMap.put(selectorPlus.getOutputName(), val); - } + final float val = key.getFloat(keyBufferPosition); + resultMap.put(selectorPlus.getOutputName(), val); } @Override public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] valuess) { - if (NullHandling.sqlCompatible() && selector.isNull()) { - valuess[columnIndex] = null; - } else { - valuess[columnIndex] = selector.getFloat(); - } + valuess[columnIndex] = selector.getFloat(); } @Override - @Nullable public Object getOnlyValue(ColumnValueSelector selector) { - if (NullHandling.sqlCompatible() && selector.isNull()) { - return null; - } return selector.getFloat(); } @Override public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBuffer keyBuffer) { - if (obj == null) { - keyBuffer.put(keyBufferPosition, (byte) 1); - keyBuffer.putFloat(keyBufferPosition + Byte.BYTES, 0f); - } else { - keyBuffer.put(keyBufferPosition, (byte) 0); - keyBuffer.putFloat(keyBufferPosition + Byte.BYTES, (Float) obj); - } + keyBuffer.putFloat(keyBufferPosition, DimensionHandlerUtils.nullToZero((Float) obj)); } @Override diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java index a50cc504d066..382ecbf2bcf4 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java @@ -59,11 +59,13 @@ public interface GroupByColumnSelectorStrategy extends ColumnSelectorStrategy * @param selectorPlus dimension info containing the key offset, value selector, and dimension spec * @param resultMap result map for the group by query being served * @param key grouping key + * @param keyBufferPosition buffer position for the grouping key */ void processValueFromGroupingKey( GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, - Map resultMap + Map resultMap, + int keyBufferPosition ); /** diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java index 8c92f74e7803..5bc9daa287af 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java @@ -19,8 +19,8 @@ package io.druid.query.groupby.epinephelinae.column; -import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; +import io.druid.segment.DimensionHandlerUtils; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -32,52 +32,34 @@ public class LongGroupByColumnSelectorStrategy implements GroupByColumnSelectorS @Override public int getGroupingKeySize() { - return Long.BYTES + Byte.BYTES; + return Long.BYTES; } @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap + GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition ) { - if (key.get(selectorPlus.getKeyBufferPosition()) == (byte) 1) { - resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultLongValue()); - } else { - final long val = key.getLong(selectorPlus.getKeyBufferPosition() + Byte.BYTES); - resultMap.put(selectorPlus.getOutputName(), val); - } + final long val = key.getLong(keyBufferPosition); + resultMap.put(selectorPlus.getOutputName(), val); } @Override public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] valuess) { - if (NullHandling.sqlCompatible() && selector.isNull()) { - valuess[columnIndex] = null; - } else { - valuess[columnIndex] = selector.getLong(); - } + valuess[columnIndex] = selector.getLong(); } @Override - @Nullable public Object getOnlyValue(ColumnValueSelector selector) { - if (NullHandling.sqlCompatible() && selector.isNull()) { - return null; - } return selector.getLong(); } @Override - public void writeToKeyBuffer(int keyBufferPosition, Object obj, ByteBuffer keyBuffer) + public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBuffer keyBuffer) { - if (obj == null) { - keyBuffer.put(keyBufferPosition, (byte) 1); - keyBuffer.putLong(keyBufferPosition + Byte.BYTES, 0L); - } else { - keyBuffer.put(keyBufferPosition, (byte) 0); - keyBuffer.putLong(keyBufferPosition + Byte.BYTES, (Long) obj); - } + keyBuffer.putLong(keyBufferPosition, DimensionHandlerUtils.nullToZero((Long) obj)); } @Override diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java new file mode 100644 index 000000000000..ab022eb4b065 --- /dev/null +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java @@ -0,0 +1,106 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.groupby.epinephelinae.column; + + +import io.druid.common.config.NullHandling; +import io.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Map; + +public class NullableValueGroupByColumnSelectorStrategy implements GroupByColumnSelectorStrategy +{ + private final GroupByColumnSelectorStrategy delegate; + + public NullableValueGroupByColumnSelectorStrategy(GroupByColumnSelectorStrategy delegate) + { + this.delegate = delegate; + } + + @Override + public int getGroupingKeySize() + { + return delegate.getGroupingKeySize() + Byte.BYTES; + } + + @Override + public void processValueFromGroupingKey( + GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition + ) + { + if (key.get(keyBufferPosition) == NullHandling.IS_NULL_BYTE) { + resultMap.put(selectorPlus.getOutputName(), null); + } else { + delegate.processValueFromGroupingKey(selectorPlus, key, resultMap, keyBufferPosition + Byte.BYTES); + } + } + + @Override + public void initColumnValues(ColumnValueSelector selector, int columnIndex, Object[] values) + { + if (selector.isNull()) { + values[columnIndex] = null; + } else { + delegate.initColumnValues(selector, columnIndex, values); + } + } + + @Override + @Nullable + public Object getOnlyValue(ColumnValueSelector selector) + { + if (selector.isNull()) { + return null; + } + return delegate.getOnlyValue(selector); + } + + @Override + public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBuffer keyBuffer) + { + if (obj == null) { + keyBuffer.put(keyBufferPosition, NullHandling.IS_NULL_BYTE); + } else { + keyBuffer.put(keyBufferPosition, NullHandling.IS_NOT_NULL_BYTE); + } + delegate.writeToKeyBuffer(keyBufferPosition + Byte.BYTES, obj, keyBuffer); + } + + @Override + public void initGroupingKeyColumnValue( + int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack + ) + { + writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); + stack[columnIndex] = 1; + } + + @Override + public boolean checkRowIndexAndAddValueToGroupingKey( + int keyBufferPosition, Object rowObj, int rowValIdx, ByteBuffer keyBuffer + ) + { + // rows from a nullable column always have a single value, multi-value is not currently supported + // this method handles row values after the first in a multivalued row, so just return false + return false; + } +} diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index cc249b84dd18..8cba28a75331 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -37,9 +37,14 @@ public int getGroupingKeySize() } @Override - public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap) + public void processValueFromGroupingKey( + GroupByColumnSelectorPlus selectorPlus, + ByteBuffer key, + Map resultMap, + int keyBufferPosition + ) { - final int id = key.getInt(selectorPlus.getKeyBufferPosition()); + final int id = key.getInt(keyBufferPosition); // GROUP_BY_MISSING_VALUE is used to indicate empty rows, which are omitted from the result map. if (id != GROUP_BY_MISSING_VALUE) { diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java index a052583ed885..dbaa036cfd1b 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.Committer; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; @@ -82,21 +83,24 @@ public void testSimpleIngestion() throws Exception // add commitMetadata.put("x", "1"); - Assert.assertEquals(1, - appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier) - .getNumRowsInSegment() + Assert.assertEquals( + 1, + appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier) + .getNumRowsInSegment() ); commitMetadata.put("x", "2"); - Assert.assertEquals(2, - appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier) - .getNumRowsInSegment() + Assert.assertEquals( + 2, + appenderator.add(IDENTIFIERS.get(0), IR("2000", "bar", 2), committerSupplier) + .getNumRowsInSegment() ); commitMetadata.put("x", "3"); - Assert.assertEquals(1, - appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier) - .getNumRowsInSegment() + Assert.assertEquals( + 1, + appenderator.add(IDENTIFIERS.get(1), IR("2000", "qux", 4), committerSupplier) + .getNumRowsInSegment() ); // getSegments @@ -172,10 +176,17 @@ public void run() appenderator.startJob(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); - //expectedSizeInBytes = 44(map overhead) + 28 (TimeAndDims overhead) + 56 (aggregator metrics) + 10 (dimsKeySize) = 138 - Assert.assertEquals(138, ((AppenderatorImpl) appenderator).getBytesInMemory(IDENTIFIERS.get(0))); + //expectedSizeInBytes = 44(map overhead) + 28 (TimeAndDims overhead) + 56 (aggregator metrics) + 10 (dimsKeySize) = 138 + 1 byte when null handling is enabled + int nullHandlingOverhead = NullHandling.sqlCompatible() ? 1 : 0; + Assert.assertEquals( + 138 + nullHandlingOverhead, + ((AppenderatorImpl) appenderator).getBytesInMemory(IDENTIFIERS.get(0)) + ); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); - Assert.assertEquals(138, ((AppenderatorImpl) appenderator).getBytesInMemory(IDENTIFIERS.get(1))); + Assert.assertEquals( + 138 + nullHandlingOverhead, + ((AppenderatorImpl) appenderator).getBytesInMemory(IDENTIFIERS.get(1)) + ); appenderator.close(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); } @@ -209,9 +220,13 @@ public void run() appenderator.startJob(); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); //expectedSizeInBytes = 44(map overhead) + 28 (TimeAndDims overhead) + 56 (aggregator metrics) + 10 (dimsKeySize) = 138 - Assert.assertEquals(138, ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory()); + int nullHandlingOverhead = NullHandling.sqlCompatible() ? 1 : 0; + Assert.assertEquals(138 + nullHandlingOverhead, ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); - Assert.assertEquals(276, ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory()); + Assert.assertEquals( + 276 + 2 * nullHandlingOverhead, + ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory() + ); appenderator.close(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); } @@ -247,10 +262,17 @@ public void run() Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "foo", 1), committerSupplier); //we still calculate the size even when ignoring it to make persist decision - Assert.assertEquals(138, ((AppenderatorImpl) appenderator).getBytesInMemory(IDENTIFIERS.get(0))); + int nullHandlingOverhead = NullHandling.sqlCompatible() ? 1 : 0; + Assert.assertEquals( + 138 + nullHandlingOverhead, + ((AppenderatorImpl) appenderator).getBytesInMemory(IDENTIFIERS.get(0)) + ); Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(1), IR("2000", "bar", 1), committerSupplier); - Assert.assertEquals(276, ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory()); + Assert.assertEquals( + 276 + 2 * nullHandlingOverhead, + ((AppenderatorImpl) appenderator).getBytesCurrentlyInMemory() + ); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close(); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); From 4f1f2ca2c0adbe52420fa799b7fc55407cf658a1 Mon Sep 17 00:00:00 2001 From: Nishant Date: Tue, 29 May 2018 18:42:48 +0530 Subject: [PATCH 33/38] review comments --- .../src/main/java/io/druid/server/lookup/LoadingLookup.java | 6 ++++-- .../src/main/java/io/druid/server/lookup/PollingLookup.java | 6 ++++-- .../java/io/druid/query/extraction/MapLookupExtractor.java | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java index f6bb7c0f2e1d..e50e3b659a89 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java @@ -67,7 +67,8 @@ public String apply(@Nullable final String key) { String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); if (keyEquivalent == null) { - // keyEquivalent is null for SQL Compatible Null Behavior + // valueEquivalent is null only for SQL Compatible Null Behavior + // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. return null; } @@ -87,7 +88,8 @@ public List unapply(@Nullable final String value) { String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); if (valueEquivalent == null) { - // valueEquivalent is null for SQL Compatible Null Behavior + // valueEquivalent is null only for SQL Compatible Null Behavior + // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. // null value maps to empty list when SQL Compatible return Collections.EMPTY_LIST; } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java index f399719a1812..ce5ce5884e13 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java @@ -113,7 +113,8 @@ public String apply(@Nullable String key) { String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); if (keyEquivalent == null) { - // keyEquivalent is null for SQL Compatible Null Behavior + // valueEquivalent is null only for SQL Compatible Null Behavior + // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. return null; } final CacheRefKeeper cacheRefKeeper = refOfCacheKeeper.get(); @@ -140,7 +141,8 @@ public List unapply(@Nullable final String value) { String valueEquivalent = NullHandling.nullToEmptyIfNeeded(value); if (valueEquivalent == null) { - // valueEquivalent is null for SQL Compatible Null Behavior + // valueEquivalent is null only for SQL Compatible Null Behavior + // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. // null value maps to empty list when SQL Compatible return Collections.EMPTY_LIST; } diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index 3cf7ef3516ea..d645b397d9a3 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -68,7 +68,8 @@ public String apply(@Nullable String key) { String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); if (keyEquivalent == null) { - // keyEquivalent is null for SQL Compatible Null Behavior + // valueEquivalent is null only for SQL Compatible Null Behavior + // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. return null; } return NullHandling.emptyToNullIfNeeded(map.get(keyEquivalent)); @@ -79,7 +80,8 @@ public List unapply(@Nullable final String value) { String valueToLookup = NullHandling.nullToEmptyIfNeeded(value); if (valueToLookup == null) { - // valueEquivalent is null for SQL Compatible Null Behavior + // valueEquivalent is null only for SQL Compatible Null Behavior + // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. // null value maps to empty list when SQL Compatible return Collections.EMPTY_LIST; } From 1c081f4a86f9faaccbeb6fdbd5e3eaaeae6f86cd Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 8 Jun 2018 08:52:35 +0530 Subject: [PATCH 34/38] review comments --- .../lookup/namespace/UriExtractionNamespace.java | 4 ++-- .../java/io/druid/server/lookup/LoadingLookup.java | 2 +- .../java/io/druid/server/lookup/PollingLookup.java | 2 +- .../io/druid/server/lookup/LoadingLookupTest.java | 4 ++-- .../AppenderatorDriverRealtimeIndexTaskTest.java | 6 +++--- .../java/io/druid/common/config/NullHandling.java | 2 +- .../java/io/druid/guice/NullHandlingModule.java | 1 - .../druid/query/aggregation/AggregateCombiner.java | 2 ++ .../io/druid/query/aggregation/AggregatorUtil.java | 9 ++++++--- .../aggregation/DoubleMaxAggregatorFactory.java | 8 ++++---- .../aggregation/DoubleMinAggregatorFactory.java | 8 ++++---- .../aggregation/DoubleSumAggregatorFactory.java | 8 ++++---- .../aggregation/FloatMaxAggregatorFactory.java | 8 ++++---- .../aggregation/FloatMinAggregatorFactory.java | 8 ++++---- .../aggregation/FloatSumAggregatorFactory.java | 8 ++++---- .../aggregation/LongMaxAggregatorFactory.java | 9 +++++---- .../aggregation/LongMinAggregatorFactory.java | 9 +++++---- .../aggregation/LongSumAggregatorFactory.java | 10 +++++----- .../aggregation/NullableAggregateCombiner.java | 9 ++++++--- .../query/aggregation/NullableAggregator.java | 4 +++- .../aggregation/NullableAggregatorFactory.java | 14 +++++++------- .../aggregation/NullableBufferAggregator.java | 10 ++++++---- .../aggregation/SimpleDoubleAggregatorFactory.java | 6 +++--- .../aggregation/SimpleFloatAggregatorFactory.java | 6 +++--- ...ardinalityAggregatorColumnSelectorStrategy.java | 2 +- .../first/DoubleFirstAggregatorFactory.java | 2 +- .../first/FloatFirstAggregatorFactory.java | 2 +- .../first/LongFirstAggregatorFactory.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 2 +- .../last/FloatLastAggregatorFactory.java | 2 +- .../last/LongLastAggregatorFactory.java | 2 +- .../druid/query/extraction/StrlenExtractionFn.java | 2 +- 32 files changed, 93 insertions(+), 80 deletions(-) diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index 8a6e9b3b3427..34036822e6fd 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -397,8 +397,8 @@ public TSVFlatDataParser( "Must specify more than one column to have a key value pair" ); final DelimitedParser delegate = new DelimitedParser( - StringUtils.emptyToNullNonDruidDataString(delimiter), - StringUtils.emptyToNullNonDruidDataString(listDelimiter), + StringUtils.emptyToNullNonDruidDataString(delimiter), + StringUtils.emptyToNullNonDruidDataString(listDelimiter), hasHeaderRow, skipHeaderRows ); diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java index e50e3b659a89..9d6d93b8efde 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java @@ -139,7 +139,7 @@ public ApplyCallable(String key) @Override public String call() { - // When SQL based null handling is disabled, + // When SQL compatible null handling is disabled, // avoid returning null and return an empty string to cache it. return NullHandling.nullToEmptyIfNeeded(dataFetcher.fetch(key)); } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java index ce5ce5884e13..c8df32a4cb7c 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java @@ -144,7 +144,7 @@ public List unapply(@Nullable final String value) // valueEquivalent is null only for SQL Compatible Null Behavior // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. // null value maps to empty list when SQL Compatible - return Collections.EMPTY_LIST; + return Collections.emptyList(); } CacheRefKeeper cacheRefKeeper = refOfCacheKeeper.get(); diff --git a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java index ce17b2299fa5..05dfd478aaf2 100644 --- a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java +++ b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/LoadingLookupTest.java @@ -44,7 +44,7 @@ public class LoadingLookupTest public void testApplyEmptyOrNull() { Assert.assertEquals(null, loadingLookup.apply(null)); - if (!NullHandling.replaceWithDefault()) { + if (NullHandling.sqlCompatible()) { // empty string should also have same behavior Assert.assertEquals(null, loadingLookup.apply("")); } @@ -54,7 +54,7 @@ public void testApplyEmptyOrNull() public void testUnapplyNull() { if (NullHandling.sqlCompatible()) { - Assert.assertEquals(Collections.EMPTY_LIST, loadingLookup.unapply(null)); + Assert.assertEquals(Collections.emptyList(), loadingLookup.unapply(null)); } else { Assert.assertEquals(null, loadingLookup.unapply(null)); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java index f4a4aab7c4f1..6537543911ce 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/AppenderatorDriverRealtimeIndexTaskTest.java @@ -495,7 +495,7 @@ public void testMaxRowsPerSegment() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2000, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(2000, sumMetric(task, null, "rows").longValue()); Assert.assertEquals(2000, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); @@ -523,7 +523,7 @@ public void testMaxRowsPerSegment() throws Exception Assert.assertEquals(TaskState.SUCCESS, taskStatus.getStatusCode()); } - @Test(timeout = 60_00000L) + @Test(timeout = 60_000L) public void testTransformSpec() throws Exception { expectPublishedSegments(2); @@ -697,7 +697,7 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); awaitHandoffs(); diff --git a/java-util/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java index ca2ddd1e19e7..6e7a6f3a5130 100644 --- a/java-util/src/main/java/io/druid/common/config/NullHandling.java +++ b/java-util/src/main/java/io/druid/common/config/NullHandling.java @@ -48,8 +48,8 @@ public class NullHandling * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. * See io.druid.guice.NullHandlingModule for details. * It does not take effect in all unit tests since we don't use Guice Injection. - * default system property is supposed to be used only in tests */ + // default system property is supposed to be used only in tests @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java index 7216ca363822..b552cb11b53b 100644 --- a/processing/src/main/java/io/druid/guice/NullHandlingModule.java +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -34,6 +34,5 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.generic", NullValueHandlingConfig.class); binder.requestStaticInjection(NullHandling.class); binder.requestStaticInjection(NullHandling.class); - } } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java index 797f44843837..8330a8d2dd17 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation; +import io.druid.guice.annotations.ExtensionPoint; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; @@ -39,6 +40,7 @@ * @see DoubleAggregateCombiner * @see ObjectAggregateCombiner */ +@ExtensionPoint public interface AggregateCombiner extends ColumnValueSelector { /** diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index b86a7a267136..1bd9a3e1f16c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -27,6 +27,9 @@ import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DoubleColumnSelector; @@ -145,7 +148,7 @@ public static Pair, List> condensedAggre return new Pair(condensedAggs, condensedPostAggs); } - public static ColumnValueSelector makeColumnValueSelectorWithFloatDefault( + public static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, final String fieldName, @@ -186,7 +189,7 @@ public boolean isNull() throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } - public static ColumnValueSelector makeColumnValueSelectorWithLongDefault( + public static BaseLongColumnValueSelector makeColumnValueSelectorWithLongDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, final String fieldName, @@ -227,7 +230,7 @@ public boolean isNull() throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); } - public static ColumnValueSelector makeColumnValueSelectorWithDoubleDefault( + public static BaseDoubleColumnValueSelector makeColumnValueSelectorWithDoubleDefault( final ColumnSelectorFactory metricFactory, final ExprMacroTable macroTable, final String fieldName, diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 964bef68c613..fa4b28b8ae1d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,7 +54,7 @@ public DoubleMaxAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseDoubleColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return getDoubleColumnSelector( metricFactory, @@ -63,13 +63,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseDoubleColumnValueSelector selector) { return new DoubleMaxAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseDoubleColumnValueSelector selector) { return new DoubleMaxBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 96b96baaac47..0bac99756e79 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -53,7 +53,7 @@ public DoubleMinAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseDoubleColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return getDoubleColumnSelector( metricFactory, @@ -62,13 +62,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseDoubleColumnValueSelector selector) { return new DoubleMinAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseDoubleColumnValueSelector selector) { return new DoubleMinBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index b3ca542ff282..cce5013f09cf 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -54,7 +54,7 @@ public DoubleSumAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseDoubleColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return getDoubleColumnSelector( metricFactory, @@ -63,13 +63,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseDoubleColumnValueSelector selector) { return new DoubleSumAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseDoubleColumnValueSelector selector) { return new DoubleSumBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index bd02b2a7a08e..bc19cb168bef 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -53,7 +53,7 @@ public FloatMaxAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseFloatColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return makeColumnValueSelectorWithFloatDefault( metricFactory, @@ -62,13 +62,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseFloatColumnValueSelector selector) { return new FloatMaxAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseFloatColumnValueSelector selector) { return new FloatMaxBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index be11112faa92..f0cb7a4ceb6b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -53,7 +53,7 @@ public FloatMinAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseFloatColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return makeColumnValueSelectorWithFloatDefault( metricFactory, @@ -62,13 +62,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseFloatColumnValueSelector selector) { return new FloatMinAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseFloatColumnValueSelector selector) { return new FloatMinBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index a34f109ee1aa..3c93793104a5 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -24,8 +24,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -53,7 +53,7 @@ public FloatSumAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseFloatColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return makeColumnValueSelectorWithFloatDefault( metricFactory, @@ -62,13 +62,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseFloatColumnValueSelector selector) { return new FloatSumAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseFloatColumnValueSelector selector) { return new FloatSumBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index ed3968cee931..7c98c254ef10 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -26,6 +26,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; +import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; @@ -39,7 +40,7 @@ /** */ -public class LongMaxAggregatorFactory extends NullableAggregatorFactory +public class LongMaxAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -72,7 +73,7 @@ public LongMaxAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseLongColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return AggregatorUtil.makeColumnValueSelectorWithLongDefault( metricFactory, @@ -84,13 +85,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseLongColumnValueSelector selector) { return new LongMaxAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseLongColumnValueSelector selector) { return new LongMaxBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 1a298e3647a8..a35c43e6267b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -26,6 +26,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; +import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; @@ -39,7 +40,7 @@ /** */ -public class LongMinAggregatorFactory extends NullableAggregatorFactory +public class LongMinAggregatorFactory extends NullableAggregatorFactory { private final String name; @@ -73,7 +74,7 @@ public LongMinAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseLongColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return AggregatorUtil.makeColumnValueSelectorWithLongDefault( metricFactory, @@ -85,13 +86,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + public Aggregator factorize(ColumnSelectorFactory metricFactory, BaseLongColumnValueSelector selector) { return new LongMinAggregator(selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseLongColumnValueSelector selector) { return new LongMinBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 3e72429848cb..83a8db23f1df 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -26,8 +26,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; +import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -39,7 +39,7 @@ /** */ -public class LongSumAggregatorFactory extends NullableAggregatorFactory +public class LongSumAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -72,7 +72,7 @@ public LongSumAggregatorFactory(String name, String fieldName) } @Override - protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) + protected BaseLongColumnValueSelector selector(ColumnSelectorFactory metricFactory) { return AggregatorUtil.makeColumnValueSelectorWithLongDefault( metricFactory, @@ -84,13 +84,13 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) } @Override - protected Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected Aggregator factorize(ColumnSelectorFactory metricFactory, BaseLongColumnValueSelector selector) { return new LongSumAggregator(selector); } @Override - protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, ColumnValueSelector selector) + protected BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory, BaseLongColumnValueSelector selector) { return new LongSumBufferAggregator(selector); } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index a1cd4a27aa54..69ddf0887037 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -19,6 +19,8 @@ package io.druid.query.aggregation; +import io.druid.guice.annotations.ExtensionPoint; +import io.druid.guice.annotations.PublicApi; import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnValueSelector; @@ -30,13 +32,14 @@ * Note that the delegate combiner is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values * will be passed to the delegate combiner. */ -public class NullableAggregateCombiner implements AggregateCombiner +@PublicApi +public final class NullableAggregateCombiner implements AggregateCombiner { private boolean isNullResult = true; - private final AggregateCombiner delegate; + private final AggregateCombiner delegate; - public NullableAggregateCombiner(AggregateCombiner delegate) + public NullableAggregateCombiner(AggregateCombiner delegate) { this.delegate = delegate; } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index c4ed13335be0..4c6083a8a0f1 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -19,6 +19,7 @@ package io.druid.query.aggregation; +import io.druid.guice.annotations.PublicApi; import io.druid.segment.BaseNullableColumnValueSelector; import javax.annotation.Nullable; @@ -29,7 +30,8 @@ * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values * will be passed to the delegate aggregator. */ -public class NullableAggregator implements Aggregator +@PublicApi +public final class NullableAggregator implements Aggregator { private final Aggregator delegate; private final BaseNullableColumnValueSelector selector; diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index 7eb8d1186912..a36a0070f48d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -22,6 +22,7 @@ import io.druid.common.config.NullHandling; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; @@ -31,12 +32,12 @@ * Support Nullable Aggregations are encouraged to extend this class. */ @ExtensionPoint -public abstract class NullableAggregatorFactory extends AggregatorFactory +public abstract class NullableAggregatorFactory extends AggregatorFactory { @Override public final Aggregator factorize(ColumnSelectorFactory metricFactory) { - ColumnValueSelector selector = selector(metricFactory); + T selector = selector(metricFactory); Aggregator aggregator = factorize(metricFactory, selector); return NullHandling.replaceWithDefault() ? aggregator : new NullableAggregator(aggregator, selector); } @@ -44,7 +45,7 @@ public final Aggregator factorize(ColumnSelectorFactory metricFactory) @Override public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - ColumnValueSelector selector = selector(metricFactory); + T selector = selector(metricFactory); BufferAggregator aggregator = factorizeBuffered(metricFactory, selector); return NullHandling.replaceWithDefault() ? aggregator : new NullableBufferAggregator(aggregator, selector); } @@ -69,7 +70,7 @@ public final int getMaxIntermediateSize() * * @see ColumnValueSelector */ - protected abstract ColumnValueSelector selector(ColumnSelectorFactory metricFactory); + protected abstract T selector(ColumnSelectorFactory metricFactory); /** * Creates an {@link Aggregator} to aggregate values from several rows, by using the provided selector. @@ -78,7 +79,7 @@ public final int getMaxIntermediateSize() * * @see Aggregator */ - protected abstract Aggregator factorize(ColumnSelectorFactory metricFactory, ColumnValueSelector selector); + protected abstract Aggregator factorize(ColumnSelectorFactory metricFactory, T selector); /** * Creates an {@link BufferAggregator} to aggregate values from several rows into a ByteBuffer. @@ -89,7 +90,7 @@ public final int getMaxIntermediateSize() */ protected abstract BufferAggregator factorizeBuffered( ColumnSelectorFactory metricFactory, - ColumnValueSelector selector + T selector ); /** @@ -101,7 +102,6 @@ protected abstract BufferAggregator factorizeBuffered( * @see AggregateCombiner * @see io.druid.segment.IndexMerger */ - @SuppressWarnings("unused") // Going to be used when https://github.com/druid-io/druid/projects/2 is complete protected abstract AggregateCombiner makeAggregateCombiner2(); /** diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 9695301bd0ac..363af1eac15e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation; import io.druid.common.config.NullHandling; +import io.druid.guice.annotations.PublicApi; import io.druid.segment.BaseNullableColumnValueSelector; import javax.annotation.Nullable; @@ -31,7 +32,8 @@ * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values * will be passed to the delegate aggregator. */ -public class NullableBufferAggregator implements BufferAggregator +@PublicApi +public final class NullableBufferAggregator implements BufferAggregator { private final BufferAggregator delegate; @@ -75,7 +77,7 @@ public Object get(ByteBuffer buf, int position) @Override public float getFloat(ByteBuffer buf, int position) { - if (isNull(buf, position)) { + if (buf.get(position) == NullHandling.IS_NULL_BYTE) { throw new IllegalStateException("Cannot return float for Null Value"); } return delegate.getFloat(buf, position + Byte.BYTES); @@ -84,7 +86,7 @@ public float getFloat(ByteBuffer buf, int position) @Override public long getLong(ByteBuffer buf, int position) { - if (isNull(buf, position)) { + if (buf.get(position) == NullHandling.IS_NULL_BYTE) { throw new IllegalStateException("Cannot return long for Null Value"); } return delegate.getLong(buf, position + Byte.BYTES); @@ -93,7 +95,7 @@ public long getLong(ByteBuffer buf, int position) @Override public double getDouble(ByteBuffer buf, int position) { - if (isNull(buf, position)) { + if (buf.get(position) == NullHandling.IS_NULL_BYTE) { throw new IllegalStateException("Cannot return double for Null Value"); } return delegate.getDouble(buf, position + Byte.BYTES); diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index e99f3ace70c6..c33bcb9ae44e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -24,8 +24,8 @@ import com.google.common.base.Preconditions; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; +import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; import javax.annotation.Nullable; @@ -34,7 +34,7 @@ import java.util.List; import java.util.Objects; -public abstract class SimpleDoubleAggregatorFactory extends NullableAggregatorFactory +public abstract class SimpleDoubleAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -61,7 +61,7 @@ public SimpleDoubleAggregatorFactory( ); } - protected ColumnValueSelector getDoubleColumnSelector(ColumnSelectorFactory metricFactory, double nullValue) + protected BaseDoubleColumnValueSelector getDoubleColumnSelector(ColumnSelectorFactory metricFactory, double nullValue) { return AggregatorUtil.makeColumnValueSelectorWithDoubleDefault( metricFactory, diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index be1752ca582b..53c7852a8787 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -23,8 +23,8 @@ import com.google.common.base.Preconditions; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; +import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; import java.util.Collections; @@ -32,7 +32,7 @@ import java.util.List; import java.util.Objects; -public abstract class SimpleFloatAggregatorFactory extends NullableAggregatorFactory +public abstract class SimpleFloatAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -57,7 +57,7 @@ public SimpleFloatAggregatorFactory( ); } - ColumnValueSelector makeColumnValueSelectorWithFloatDefault( + BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefault( ColumnSelectorFactory metricFactory, float nullValue ) diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java index f27a40fcceba..e133ba61061f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java @@ -52,7 +52,7 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) // SQL standard spec does not count null values, // Skip counting null values when we are not replacing null with default value. // A special value for null in case null handling is configured to use empty string for null. - if (!NullHandling.replaceWithDefault() && !hasNonNullValue && value != null) { + if (NullHandling.sqlCompatible() && !hasNonNullValue && value != null) { hasNonNullValue = true; } values[i] = nullToSpecial(value); diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 3330b54a0852..209352a490e2 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -46,7 +46,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleFirstAggregatorFactory extends NullableAggregatorFactory +public class DoubleFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 4687b65cd2a0..af35d3699083 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -46,7 +46,7 @@ import java.util.Map; import java.util.Objects; -public class FloatFirstAggregatorFactory extends NullableAggregatorFactory +public class FloatFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index cee1c18e178b..cafb33f8ab23 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -44,7 +44,7 @@ import java.util.List; import java.util.Map; -public class LongFirstAggregatorFactory extends NullableAggregatorFactory +public class LongFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Longs.compare( ((SerializablePair) o1).rhs, diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 8edc224d544c..4e5344e8f4d7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -46,7 +46,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleLastAggregatorFactory extends NullableAggregatorFactory +public class DoubleLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index c83f491dd9ab..5740a18b413a 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -46,7 +46,7 @@ import java.util.Map; import java.util.Objects; -public class FloatLastAggregatorFactory extends NullableAggregatorFactory +public class FloatLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index d927fdfc036f..70c5a970f6a4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -46,7 +46,7 @@ import java.util.Map; import java.util.Objects; -public class LongLastAggregatorFactory extends NullableAggregatorFactory +public class LongLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; private final String name; diff --git a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java index 780f1ba7afe4..f2379089db93 100644 --- a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java @@ -41,7 +41,7 @@ public static StrlenExtractionFn instance() @Override public String apply(@Nullable String value) { - if (!NullHandling.replaceWithDefault() && value == null) { + if (NullHandling.sqlCompatible() && value == null) { return null; } return String.valueOf(value == null ? 0 : value.length()); From 00e14b7158f37733912122a5261f5c73d32db0cf Mon Sep 17 00:00:00 2001 From: Nishant Date: Mon, 18 Jun 2018 16:33:03 -0700 Subject: [PATCH 35/38] review comments --- .../src/main/java/io/druid/common/config/NullHandling.java | 1 - .../query/aggregation/post/DoubleGreatestPostAggregator.java | 2 +- .../druid/query/aggregation/post/DoubleLeastPostAggregator.java | 2 +- .../query/aggregation/post/LongGreatestPostAggregator.java | 2 +- .../druid/query/aggregation/post/LongLeastPostAggregator.java | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/java-util/src/main/java/io/druid/common/config/NullHandling.java b/java-util/src/main/java/io/druid/common/config/NullHandling.java index 6e7a6f3a5130..af5acfbbcf9a 100644 --- a/java-util/src/main/java/io/druid/common/config/NullHandling.java +++ b/java-util/src/main/java/io/druid/common/config/NullHandling.java @@ -49,7 +49,6 @@ public class NullHandling * See io.druid.guice.NullHandlingModule for details. * It does not take effect in all unit tests since we don't use Guice Injection. */ - // default system property is supposed to be used only in tests @Inject private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 7f74364e7269..02ec9627ec20 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -38,7 +38,7 @@ public class DoubleGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Number::doubleValue)); + private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Number::doubleValue)); private final String name; private final List fields; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index 7eff7073f2e7..b4131172759b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -38,7 +38,7 @@ public class DoubleLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparator.nullsLast(Comparator.comparingDouble(Number::doubleValue)); + private static final Comparator COMPARATOR = Comparator.nullsLast(Comparator.comparingDouble(Number::doubleValue)); private final String name; private final List fields; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index e1c9c3e43780..f9d522fa2092 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -38,7 +38,7 @@ public class LongGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Number::longValue)); + private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(Number::longValue)); private final String name; private final List fields; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index 1a9ef10bf033..49cc5ed49649 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -38,7 +38,7 @@ public class LongLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparator.nullsLast(Comparator.comparingLong(Number::longValue)); + private static final Comparator COMPARATOR = Comparator.nullsLast(Comparator.comparingLong(Number::longValue)); private final String name; private final List fields; From ef0e725b44842d19652cc1ba3789fa590e00ca2c Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 22 Jun 2018 15:47:56 -0700 Subject: [PATCH 36/38] Review comments --- .../post/DoubleGreatestPostAggregator.java | 2 +- .../post/DoubleLeastPostAggregator.java | 2 +- .../post/LongGreatestPostAggregator.java | 2 +- .../post/LongLeastPostAggregator.java | 2 +- .../expression/TimestampCeilExprMacro.java | 5 +++++ .../expression/TimestampFloorExprMacro.java | 7 ++++++- .../expression/TimestampFormatExprMacro.java | 5 +++++ .../query/extraction/MapLookupExtractor.java | 17 ++++++++------- .../io/druid/query/filter/ValueGetter.java | 1 + .../epinephelinae/RowBasedGrouperHelper.java | 21 +++++++++++-------- .../DoubleGroupByColumnSelectorStrategy.java | 11 ++++++++-- .../FloatGroupByColumnSelectorStrategy.java | 11 ++++++++-- .../column/GroupByColumnSelectorStrategy.java | 2 +- .../LongGroupByColumnSelectorStrategy.java | 11 ++++++++-- ...bleValueGroupByColumnSelectorStrategy.java | 11 ++++++++-- .../groupby/having/EqualToHavingSpec.java | 2 +- .../groupby/having/GreaterThanHavingSpec.java | 2 +- .../having/HavingSpecMetricComparator.java | 6 +----- .../groupby/having/LessThanHavingSpec.java | 2 +- .../groupby/orderby/DefaultLimitSpec.java | 15 +++++++++++-- 20 files changed, 96 insertions(+), 41 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 02ec9627ec20..4e9989ffc0e1 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -79,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal, retVal) > 0) { + if (nextVal != null && (retVal == null || COMPARATOR.compare(nextVal, retVal) > 0)) { if (nextVal instanceof Double) { retVal = (Double) nextVal; } else { diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index b4131172759b..da59c261dc50 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -79,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal, retVal) < 0) { + if (nextVal != null && (retVal == null || COMPARATOR.compare(nextVal, retVal) < 0)) { if (nextVal instanceof Double) { retVal = (Double) nextVal; } else { diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index f9d522fa2092..4c02961fb318 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -79,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal, retVal) > 0) { + if (nextVal != null && (retVal == null || COMPARATOR.compare(nextVal, retVal) > 0)) { if (nextVal instanceof Long) { retVal = (Long) nextVal; } else { diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index 49cc5ed49649..de1a2cd6bd3b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -79,7 +79,7 @@ public Object compute(Map values) while (fieldsIter.hasNext()) { Number nextVal = ((Number) fieldsIter.next().compute(values)); // Ignore NULL values and return the greatest out of non-null values. - if (nextVal != null && COMPARATOR.compare(nextVal, retVal) < 0) { + if (nextVal != null && (retVal == null || COMPARATOR.compare(nextVal, retVal) < 0)) { if (nextVal instanceof Long) { retVal = (Long) nextVal; } else { diff --git a/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java index 48ce083e7300..0bba43ebcf41 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java @@ -67,6 +67,11 @@ public TimestampCeilExpr(final List args) @Override public ExprEval eval(final ObjectBinding bindings) { + ExprEval eval = arg.eval(bindings); + if (eval.isNumericNull()) { + // Return null if the argument if null. + return ExprEval.of(null); + } return ExprEval.of(granularity.bucketEnd(DateTimes.utc(arg.eval(bindings).asLong())).getMillis()); } diff --git a/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java index 984fb8592c08..bde9339eb498 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java @@ -92,7 +92,12 @@ public PeriodGranularity getGranularity() @Override public ExprEval eval(final ObjectBinding bindings) { - return ExprEval.of(granularity.bucketStart(DateTimes.utc(arg.eval(bindings).asLong())).getMillis()); + ExprEval eval = arg.eval(bindings); + if (eval.isNumericNull()) { + // Return null if the argument if null. + return ExprEval.of(null); + } + return ExprEval.of(granularity.bucketStart(DateTimes.utc(eval.asLong())).getMillis()); } @Override diff --git a/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java index 71af26d8633e..d4063c49ca0d 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java @@ -74,6 +74,11 @@ class TimestampFormatExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { + ExprEval eval = arg.eval(bindings); + if (eval.isNumericNull()) { + // Return null if the argument if null. + return ExprEval.of(null); + } return ExprEval.of(formatter.print(arg.eval(bindings).asLong())); } diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index d645b397d9a3..2585e51e7033 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -26,8 +26,6 @@ import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.lookup.LookupExtractor; @@ -38,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @JsonTypeName("map") public class MapLookupExtractor extends LookupExtractor @@ -68,7 +67,7 @@ public String apply(@Nullable String key) { String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key); if (keyEquivalent == null) { - // valueEquivalent is null only for SQL Compatible Null Behavior + // keyEquivalent is null only for SQL Compatible Null Behavior // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. return null; } @@ -80,14 +79,16 @@ public List unapply(@Nullable final String value) { String valueToLookup = NullHandling.nullToEmptyIfNeeded(value); if (valueToLookup == null) { - // valueEquivalent is null only for SQL Compatible Null Behavior + // valueToLookup is null only for SQL Compatible Null Behavior // otherwise null will be replaced with empty string in nullToEmptyIfNeeded above. // null value maps to empty list when SQL Compatible - return Collections.EMPTY_LIST; + return Collections.emptyList(); } - - return Lists.newArrayList(Maps.filterKeys(map, key -> map.get(key).equals(valueToLookup)).keySet()); - + return map.entrySet() + .stream() + .filter(entry -> entry.getValue().equals(valueToLookup)) + .map(entry -> entry.getKey()) + .collect(Collectors.toList()); } @Override diff --git a/processing/src/main/java/io/druid/query/filter/ValueGetter.java b/processing/src/main/java/io/druid/query/filter/ValueGetter.java index c478ce971b37..ff4caf36edc6 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueGetter.java +++ b/processing/src/main/java/io/druid/query/filter/ValueGetter.java @@ -29,6 +29,7 @@ public interface ValueGetter // converted to strings. We should also add functions // for these and modify ColumnComparisonFilter to handle // comparing Long and Float columns to eachother. + // Returns null when the underlying Long/Float value is null. @Nullable String[] get(); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index df0275acd171..e3f908f566f7 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -1397,12 +1397,15 @@ private RowBasedKeySerdeHelper makeNullHandlingNumericserdeHelper( ) { if (NullHandling.sqlCompatible()) { - return new NullableRowBasedKeySerdeHelper(makeNumericSerdeHelper( - valueType, - keyBufferPosition + Byte.BYTES, - pushLimitDown, - stringComparator - ), keyBufferPosition); + return new NullableRowBasedKeySerdeHelper( + makeNumericSerdeHelper( + valueType, + keyBufferPosition + Byte.BYTES, + pushLimitDown, + stringComparator + ), + keyBufferPosition + ); } else { return makeNumericSerdeHelper(valueType, keyBufferPosition, pushLimitDown, stringComparator); } @@ -1723,8 +1726,8 @@ private class NullableRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper BufferComparator delegateBufferComparator = this.delegate.getBufferComparator(); this.comparator = (lhsBuffer, rhsBuffer, lhsPosition, rhsPosition) -> { boolean isLhsNull = (lhsBuffer.get(lhsPosition + keyBufferPosition) == NullHandling.IS_NULL_BYTE); - boolean isrhsNull = (rhsBuffer.get(rhsPosition + keyBufferPosition) == NullHandling.IS_NULL_BYTE); - if (isLhsNull && isrhsNull) { + boolean isRhsNull = (rhsBuffer.get(rhsPosition + keyBufferPosition) == NullHandling.IS_NULL_BYTE); + if (isLhsNull && isRhsNull) { // Both are null return 0; } @@ -1733,7 +1736,7 @@ private class NullableRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper return -1; } // only rhs is null - if (isrhsNull) { + if (isRhsNull) { return 1; } return delegateBufferComparator.compare( diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java index 972d519bec02..bcbd50b71fa0 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DoubleGroupByColumnSelectorStrategy.java @@ -37,7 +37,10 @@ public int getGroupingKeySize() @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition + GroupByColumnSelectorPlus selectorPlus, + ByteBuffer key, + Map resultMap, + int keyBufferPosition ) { final double val = key.getDouble(keyBufferPosition); @@ -64,7 +67,11 @@ public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBu @Override public void initGroupingKeyColumnValue( - int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack + int keyBufferPosition, + int columnIndex, + Object rowObj, + ByteBuffer keyBuffer, + int[] stack ) { writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java index 6a619126462f..3b5996e0ebd2 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/FloatGroupByColumnSelectorStrategy.java @@ -37,7 +37,10 @@ public int getGroupingKeySize() @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition + GroupByColumnSelectorPlus selectorPlus, + ByteBuffer key, + Map resultMap, + int keyBufferPosition ) { final float val = key.getFloat(keyBufferPosition); @@ -64,7 +67,11 @@ public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBu @Override public void initGroupingKeyColumnValue( - int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack + int keyBufferPosition, + int columnIndex, + Object rowObj, + ByteBuffer keyBuffer, + int[] stack ) { writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java index 382ecbf2bcf4..9fd2450e8cae 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/GroupByColumnSelectorStrategy.java @@ -59,7 +59,7 @@ public interface GroupByColumnSelectorStrategy extends ColumnSelectorStrategy * @param selectorPlus dimension info containing the key offset, value selector, and dimension spec * @param resultMap result map for the group by query being served * @param key grouping key - * @param keyBufferPosition buffer position for the grouping key + * @param keyBufferPosition buffer position for the grouping key, added to support chaining multiple {@link ColumnSelectorStrategy} */ void processValueFromGroupingKey( GroupByColumnSelectorPlus selectorPlus, diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java index 5bc9daa287af..515b7ccb93fa 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/LongGroupByColumnSelectorStrategy.java @@ -37,7 +37,10 @@ public int getGroupingKeySize() @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition + GroupByColumnSelectorPlus selectorPlus, + ByteBuffer key, + Map resultMap, + int keyBufferPosition ) { final long val = key.getLong(keyBufferPosition); @@ -64,7 +67,11 @@ public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBu @Override public void initGroupingKeyColumnValue( - int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack + int keyBufferPosition, + int columnIndex, + Object rowObj, + ByteBuffer keyBuffer, + int[] stack ) { writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java index ab022eb4b065..576ff3010aa1 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/NullableValueGroupByColumnSelectorStrategy.java @@ -44,7 +44,10 @@ public int getGroupingKeySize() @Override public void processValueFromGroupingKey( - GroupByColumnSelectorPlus selectorPlus, ByteBuffer key, Map resultMap, int keyBufferPosition + GroupByColumnSelectorPlus selectorPlus, + ByteBuffer key, + Map resultMap, + int keyBufferPosition ) { if (key.get(keyBufferPosition) == NullHandling.IS_NULL_BYTE) { @@ -87,7 +90,11 @@ public void writeToKeyBuffer(int keyBufferPosition, @Nullable Object obj, ByteBu @Override public void initGroupingKeyColumnValue( - int keyBufferPosition, int columnIndex, Object rowObj, ByteBuffer keyBuffer, int[] stack + int keyBufferPosition, + int columnIndex, + Object rowObj, + ByteBuffer keyBuffer, + int[] stack ) { writeToKeyBuffer(keyBufferPosition, rowObj, keyBuffer); diff --git a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java index 7f1127d92519..d1674b18dd82 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java @@ -72,7 +72,7 @@ public boolean eval(Row row) if (metricVal == null || value == null) { return metricVal == null && value == null; } - return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) == 0; + return HavingSpecMetricComparator.compare(aggregationName, value, aggregators, metricVal) == 0; } /** diff --git a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java index 06fe975aea6d..9b9755151631 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java @@ -72,7 +72,7 @@ public boolean eval(Row row) if (metricVal == null || value == null) { return false; } - return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) > 0; + return HavingSpecMetricComparator.compare(aggregationName, value, aggregators, metricVal) > 0; } /** diff --git a/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java b/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java index 1b041dc1be5a..d6e18a7f55b4 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java +++ b/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java @@ -21,7 +21,6 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; -import io.druid.data.input.Row; import io.druid.java.util.common.ISE; import io.druid.query.aggregation.AggregatorFactory; @@ -35,11 +34,8 @@ class HavingSpecMetricComparator { static final Pattern LONG_PAT = Pattern.compile("[-|+]?\\d+"); - static int compare(Row row, String aggregationName, Number value, Map aggregators) + static int compare(String aggregationName, Number value, Map aggregators, Object metricValueObj) { - - Object metricValueObj = row.getRaw(aggregationName); - if (metricValueObj != null) { if (aggregators != null && aggregators.containsKey(aggregationName)) { metricValueObj = aggregators.get(aggregationName).finalizeComputation(metricValueObj); diff --git a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java index 6ab9747639c2..6729bf19924e 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java @@ -70,7 +70,7 @@ public boolean eval(Row row) if (metricVal == null || value == null) { return false; } - return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) < 0; + return HavingSpecMetricComparator.compare(aggregationName, value, aggregators, metricVal) < 0; } /** diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index 557d71e1f97f..b79a63781938 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -30,6 +30,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.data.input.Row; import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularities; @@ -269,12 +270,22 @@ public int compare(Row left, Row right) private Ordering metricOrdering(final String column, final Comparator comparator) { - return Ordering.from(Comparator.comparing((Row row) -> row.getRaw(column), Comparator.nullsFirst(comparator))); + if (NullHandling.sqlCompatible()) { + return Ordering.from(Comparator.comparing((Row row) -> row.getRaw(column), Comparator.nullsFirst(comparator))); + } else { + return Ordering.from(Comparator.comparing((Row row) -> row.getRaw(column), Comparator.nullsLast(comparator))); + } } private Ordering dimensionOrdering(final String dimension, final StringComparator comparator) { - return Ordering.from(Comparator.comparing((Row row) -> row.getDimension(dimension).isEmpty() ? null : row.getDimension(dimension).get(0), Comparator.nullsFirst(comparator))); + return Ordering.from( + Comparator.comparing( + (Row row) -> row.getDimension(dimension).isEmpty() + ? null + : row.getDimension(dimension).get(0), + Comparator.nullsFirst(comparator) + )); } @Override From cd8b1249dbfac182fc9c06e6130fb0528db6f34d Mon Sep 17 00:00:00 2001 From: Nishant Date: Fri, 22 Jun 2018 17:02:23 -0700 Subject: [PATCH 37/38] add comments to code --- .../io/druid/query/aggregation/NullableAggregateCombiner.java | 2 +- .../java/io/druid/query/aggregation/NullableAggregator.java | 2 +- .../io/druid/query/aggregation/NullableBufferAggregator.java | 2 +- .../query/groupby/epinephelinae/RowBasedGrouperHelper.java | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java index 6c024c414aee..f233defb4c3c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -29,7 +29,7 @@ * The result of a NullableAggregateCombiner will be null if all the values to be combined are null values or no values are combined at all. * If any of the value is non-null, the result would be the value of the delegate combiner. * Note that the delegate combiner is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values - * will be passed to the delegate combiner. + * will be passed to the delegate combiner. This class is only used when SQL compatible null handling is enabled. */ @PublicApi public final class NullableAggregateCombiner implements AggregateCombiner diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 4c6083a8a0f1..9ec1871858a9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -28,7 +28,7 @@ * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values - * will be passed to the delegate aggregator. + * will be passed to the delegate aggregator. This class is only used when SQL compatible null handling is enabled. */ @PublicApi public final class NullableAggregator implements Aggregator diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java index 363af1eac15e..ea66f7f5cf50 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -30,7 +30,7 @@ * The result of a NullableBufferAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values - * will be passed to the delegate aggregator. + * will be passed to the delegate aggregator. This class is only used when SQL compatible null handling is enabled. */ @PublicApi public final class NullableBufferAggregator implements BufferAggregator diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index e3f908f566f7..db8706f7620d 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -1713,6 +1713,7 @@ public BufferComparator getBufferComparator() } } + // This class is only used when SQL compatible null handling is enabled. private class NullableRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper { private final RowBasedKeySerdeHelper delegate; From 7074b6906755ce2b8aac9349cb69a983d492ec23 Mon Sep 17 00:00:00 2001 From: Nishant Date: Sat, 7 Jul 2018 00:28:17 +0530 Subject: [PATCH 38/38] review comments --- .../io/druid/query/aggregation/NullableAggregator.java | 9 +++++---- .../query/aggregation/NullableAggregatorFactory.java | 8 ++++---- .../aggregation/post/DoubleGreatestPostAggregator.java | 4 +++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java index 9ec1871858a9..6b8ddc3dcece 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -25,10 +25,11 @@ import javax.annotation.Nullable; /** - * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. - * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. - * Note that the delegate aggregator is not required to perform check for {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values - * will be passed to the delegate aggregator. This class is only used when SQL compatible null handling is enabled. + * The result of a NullableAggregator will be null if all the values to be aggregated are null values + * or no values are aggregated at all. If any of the value is non-null, the result would be the aggregated + * value of the delegate aggregator. Note that the delegate aggregator is not required to perform check for + * {@link BaseNullableColumnValueSelector#isNull()} on the selector as only non-null values will be passed + * to the delegate aggregator. This class is only used when SQL compatible null handling is enabled. */ @PublicApi public final class NullableAggregator implements Aggregator diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java index a36a0070f48d..623ca2f021cb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -94,10 +94,10 @@ protected abstract BufferAggregator factorizeBuffered( ); /** - * Creates an {@link AggregateCombiner} to fold rollup aggregation results from serveral "rows" of different indexes during - * index merging. AggregateCombiner implements the same logic as {@link #combine}, with the difference that it uses - * {@link ColumnValueSelector} and it's subinterfaces to get inputs and implements {@code - * ColumnValueSelector} to provide output. + * Creates an {@link AggregateCombiner} to fold rollup aggregation results from serveral "rows" of different indexes + * during index merging. AggregateCombiner implements the same logic as {@link #combine}, with the difference that it + * uses {@link ColumnValueSelector} and it's subinterfaces to get inputs and implements {@code ColumnValueSelector} + * to provide output. * * @see AggregateCombiner * @see io.druid.segment.IndexMerger diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 4e9989ffc0e1..6f67c3a9f3eb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -38,7 +38,9 @@ public class DoubleGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Number::doubleValue)); + private static final Comparator COMPARATOR = Comparator.nullsFirst( + Comparator.comparingDouble(Number::doubleValue) + ); private final String name; private final List fields;