diff --git a/.travis.yml b/.travis.yml index 4fb64a7bf9af..bc0b0f1fa016 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,8 +50,38 @@ matrix: install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing,!server' + # processing module test with SQL Compatible Null Handling + - sudo: false + env: + - NAME="processing module test with SQL Compatible Null Handling" + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl processing + + # server module test with SQL Compatible Null Handling + - sudo: false + env: + - NAME="server module test with SQL Compatible Null Handling" + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -Ddruid.generic.useDefaultValueForNull=false -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + # Server module test is run without the parallel-test option because it's memory sensitive and often fails with that option. + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -Ddruid.generic.useDefaultValueForNull=false -B -pl server + + # other modules test with SQL Compatible Null Handling + - sudo: false + env: + - NAME="other modules test with SQL Compatible Null Handling" + install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B + before_script: + - unset _JAVA_OPTIONS + + script: echo "MAVEN_OPTS='-Xmx512m'" > ~/.mavenrc && mvn test -Ddruid.generic.useDefaultValueForNull=false -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing,!server' + # run integration tests - sudo: required services: diff --git a/api/src/main/java/io/druid/data/input/MapBasedRow.java b/api/src/main/java/io/druid/data/input/MapBasedRow.java index 5a7c4db03737..312a13ff87fc 100644 --- a/api/src/main/java/io/druid/data/input/MapBasedRow.java +++ b/api/src/main/java/io/druid/data/input/MapBasedRow.java @@ -25,6 +25,7 @@ import io.druid.java.util.common.DateTimes; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; import java.util.Map; @@ -80,6 +81,7 @@ public List getDimension(String dimension) } @Override + @Nullable public Object getRaw(String dimension) { return event.get(dimension); diff --git a/api/src/main/java/io/druid/data/input/Row.java b/api/src/main/java/io/druid/data/input/Row.java index b8ed24c803d1..d63730b023b0 100644 --- a/api/src/main/java/io/druid/data/input/Row.java +++ b/api/src/main/java/io/druid/data/input/Row.java @@ -24,6 +24,7 @@ import io.druid.guice.annotations.PublicApi; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; /** @@ -71,13 +72,13 @@ public interface Row extends Comparable * * @return the value of the provided column name */ + @Nullable Object getRaw(String dimension); /** - * Returns the metric column value for the given column name. This method is different from {@link #getRaw} in two - * aspects: - * 1. If the column is absent in the row, numeric zero is returned, rather than null. - * 2. If the column has string value, an attempt is made to parse this value as a number. + * Returns the metric column value for the given column name. This method is different from {@link #getRaw} + * when column has string value, an attempt is made to parse this value as a number. */ + @Nullable Number getMetric(String metric); } diff --git a/api/src/main/java/io/druid/data/input/Rows.java b/api/src/main/java/io/druid/data/input/Rows.java index 0ef09e9fa246..7448e10e373d 100644 --- a/api/src/main/java/io/druid/data/input/Rows.java +++ b/api/src/main/java/io/druid/data/input/Rows.java @@ -26,6 +26,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.parsers.ParseException; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -92,10 +93,11 @@ public static List objectToStrings(final Object inputValue) * @throws NullPointerException if the string is null * @throws ParseException if the column cannot be converted to a number */ - public static Number objectToNumber(final String name, final Object inputValue) + @Nullable + public static Number objectToNumber(final String name, @Nullable final Object inputValue) { if (inputValue == null) { - return Rows.LONG_ZERO; + return null; } if (inputValue instanceof Number) { diff --git a/api/src/test/java/io/druid/data/input/MapBasedRowTest.java b/api/src/test/java/io/druid/data/input/MapBasedRowTest.java index b6b4d5a9840f..1faeb6927be4 100644 --- a/api/src/test/java/io/druid/data/input/MapBasedRowTest.java +++ b/api/src/test/java/io/druid/data/input/MapBasedRowTest.java @@ -41,7 +41,7 @@ public void testGetLongMetricFromString() .put("k6", "+9223372036854775802") .build() ); - + Assert.assertEquals(-1.2, row.getMetric("k0")); Assert.assertEquals(1.23, row.getMetric("k1")); Assert.assertEquals(1.8, row.getMetric("k2")); @@ -49,5 +49,6 @@ public void testGetLongMetricFromString() Assert.assertEquals(9223372036854775806L, row.getMetric("k4")); Assert.assertEquals(-9223372036854775807L, row.getMetric("k5")); Assert.assertEquals(9223372036854775802L, row.getMetric("k6")); + Assert.assertEquals(null, row.getMetric("k9")); } } diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java index bd7e8415495e..67864e161e24 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java @@ -610,8 +610,9 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { + //CHECKSTYLE.OFF: Regexp final String valueOrNull = Strings.emptyToNull(value); - + //CHECKSTYLE.ON: Regexp final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override diff --git a/codestyle/checkstyle.xml b/codestyle/checkstyle.xml index 5596e3f91054..0379e71f0119 100644 --- a/codestyle/checkstyle.xml +++ b/codestyle/checkstyle.xml @@ -162,5 +162,15 @@ + + + + + + + + + + diff --git a/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 index c3f88029b031..c3547e70d2ed 100644 --- a/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 +++ b/common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4 @@ -1,6 +1,7 @@ grammar Expr; -expr : ('-'|'!') expr # unaryOpExpr +expr : 'null' # null + | ('-'|'!') expr # unaryOpExpr | expr '^' expr # powOpExpr | expr ('*'|'/'|'%') expr # mulDivModuloExpr | expr ('+'|'-') expr # addSubExpr diff --git a/common/src/main/java/io/druid/common/config/NullHandling.java b/common/src/main/java/io/druid/common/config/NullHandling.java new file mode 100644 index 000000000000..75df6e980e73 --- /dev/null +++ b/common/src/main/java/io/druid/common/config/NullHandling.java @@ -0,0 +1,106 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.common.config; + +import com.google.common.base.Strings; +import com.google.inject.Inject; + +import javax.annotation.Nullable; + +/** + * Helper class for NullHandling. This class is used to switch between SQL compatible Null Handling behavior + * introduced as part of {@link https://github.com/druid-io/druid/issues/4349} and the old druid behavior + * where null values are replaced with default values e.g Null Strings are replaced with empty values. + */ +public class NullHandling +{ + private static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull"; + + /** + * use these values to ensure that {@link NullHandling#defaultDoubleValue()}, + * {@link NullHandling#defaultFloatValue()} , {@link NullHandling#defaultFloatValue()} + * return the same boxed object when returning a constant zero + */ + public static final Double ZERO_DOUBLE = 0.0d; + public static final Float ZERO_FLOAT = 0.0f; + public static final Long ZERO_LONG = 0L; + + /** + * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code. + * See {@link io.druid.guice.NullHandlingModule} for details. + * It does not take effect in all unit tests since we don't use Guice Injection. + * For tests default system property is supposed to be used only in tests + */ + @Inject + private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig( + Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true")) + ); + + public static boolean useDefaultValuesForNull() + { + return INSTANCE.isUseDefaultValuesForNull(); + } + + @Nullable + public static String nullToEmptyIfNeeded(@Nullable String value) + { + //CHECKSTYLE.OFF: Regexp + return useDefaultValuesForNull() ? Strings.nullToEmpty(value) : value; + //CHECKSTYLE.ON: Regexp + } + + @Nullable + public static String emptyToNullIfNeeded(@Nullable String value) + { + //CHECKSTYLE.OFF: Regexp + return useDefaultValuesForNull() ? Strings.emptyToNull(value) : value; + //CHECKSTYLE.ON: Regexp + } + + @Nullable + public static String defaultStringValue() + { + return useDefaultValuesForNull() ? "" : null; + } + + @Nullable + public static Long defaultLongValue() + { + return useDefaultValuesForNull() ? ZERO_LONG : null; + } + + @Nullable + public static Float defaultFloatValue() + { + return useDefaultValuesForNull() ? ZERO_FLOAT : null; + } + + @Nullable + public static Double defaultDoubleValue() + { + return useDefaultValuesForNull() ? ZERO_DOUBLE : null; + } + + public static boolean isNullOrEquivalent(@Nullable String value) + { + return INSTANCE.isUseDefaultValuesForNull() ? Strings.isNullOrEmpty(value) : value == null; + } + +} diff --git a/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java new file mode 100644 index 000000000000..fdd1638b510e --- /dev/null +++ b/common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.common.config; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class NullValueHandlingConfig +{ + + @JsonProperty("useDefaultValueForNull") + private final boolean useDefaultValuesForNull; + + @JsonCreator + public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull) + { + this.useDefaultValuesForNull = useDefaultValuesForNull == null + ? true + : useDefaultValuesForNull; + } + + public boolean isUseDefaultValuesForNull() + { + return useDefaultValuesForNull; + } +} diff --git a/common/src/main/java/io/druid/io/Channels.java b/common/src/main/java/io/druid/io/Channels.java index 141da64d0fc8..876fcd898586 100644 --- a/common/src/main/java/io/druid/io/Channels.java +++ b/common/src/main/java/io/druid/io/Channels.java @@ -25,7 +25,6 @@ public final class Channels { - public static void writeFully(WritableByteChannel channel, ByteBuffer src) throws IOException { while (src.remaining() > 0) { diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index 913f40244bc8..a6ed91e4e0f3 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -20,9 +20,9 @@ package io.druid.math.expr; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.math.LongMath; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -124,7 +124,7 @@ class StringExpr extends ConstantExpr public StringExpr(String value) { - this.value = Strings.emptyToNull(value); + this.value = NullHandling.emptyToNullIfNeeded(value); } @Nullable @@ -362,6 +362,13 @@ public ExprEval eval(ObjectBinding bindings) { ExprEval leftVal = left.eval(bindings); ExprEval rightVal = right.eval(bindings); + + // Result of any Binary expressions is null if any of the argument is null. + // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec. + if (!NullHandling.useDefaultValuesForNull() && (leftVal.isNull() || rightVal.isNull())) { + return ExprEval.of(null); + } + if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) { return evalString(leftVal.asString(), rightVal.asString()); } else if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) { @@ -491,7 +498,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(Strings.nullToEmpty(left) + Strings.nullToEmpty(right)); + return ExprEval.of(NullHandling.nullToEmptyIfNeeded(left) + + NullHandling.nullToEmptyIfNeeded(right)); } @Override @@ -698,3 +706,4 @@ public ExprEval eval(ObjectBinding bindings) return leftVal.asBoolean() ? leftVal : right.eval(bindings); } } + diff --git a/common/src/main/java/io/druid/math/expr/ExprEval.java b/common/src/main/java/io/druid/math/expr/ExprEval.java index 972ce342fe54..2aa724c6081b 100644 --- a/common/src/main/java/io/druid/math/expr/ExprEval.java +++ b/common/src/main/java/io/druid/math/expr/ExprEval.java @@ -20,12 +20,14 @@ package io.druid.math.expr; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.primitives.Doubles; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.IAE; +import javax.annotation.Nullable; + /** */ public abstract class ExprEval @@ -50,7 +52,7 @@ public static ExprEval of(double doubleValue) return new DoubleExprEval(doubleValue); } - public static ExprEval of(String stringValue) + public static ExprEval of(@Nullable String stringValue) { return new StringExprEval(stringValue); } @@ -108,6 +110,7 @@ public boolean isNull() public abstract double asDouble(); + @Nullable public String asString() { return value == null ? null : String.valueOf(value); @@ -228,9 +231,9 @@ public Expr toExpr() private static class StringExprEval extends ExprEval { - private StringExprEval(String value) + private StringExprEval(@Nullable String value) { - super(Strings.emptyToNull(value)); + super(NullHandling.emptyToNullIfNeeded(value)); } @Override diff --git a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java index 270cd870c552..cb4eab179a3b 100644 --- a/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java +++ b/common/src/main/java/io/druid/math/expr/ExprListenerImpl.java @@ -335,4 +335,10 @@ public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx) nodes.put(ctx, args); } + + @Override + public void exitNull(ExprParser.NullContext ctx) + { + nodes.put(ctx, new StringExpr(null)); + } } diff --git a/common/src/main/java/io/druid/math/expr/Function.java b/common/src/main/java/io/druid/math/expr/Function.java index 0e067583cfcf..9a589ebc9fd3 100644 --- a/common/src/main/java/io/druid/math/expr/Function.java +++ b/common/src/main/java/io/druid/math/expr/Function.java @@ -19,7 +19,7 @@ package io.druid.math.expr; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; @@ -74,6 +74,9 @@ abstract class SingleParamMath extends SingleParam @Override protected final ExprEval eval(ExprEval param) { + if (!NullHandling.useDefaultValuesForNull() && param.isNull()) { + return ExprEval.of(null); + } if (param.type() == ExprType.LONG) { return eval(param.asLong()); } else if (param.type() == ExprType.DOUBLE) { @@ -896,10 +899,20 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(null); } else { // Pass first argument in to the constructor to provide StringBuilder a little extra sizing hint. - final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString())); + String first = NullHandling.nullToEmptyIfNeeded(args.get(0).eval(bindings).asString()); + if (first == null) { + // Result of concatenation is null if any of the Values is null. + // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. + return ExprEval.of(null); + } + final StringBuilder builder = new StringBuilder(first); for (int i = 1; i < args.size(); i++) { - final String s = args.get(i).eval(bindings).asString(); - if (s != null) { + final String s = NullHandling.nullToEmptyIfNeeded(args.get(i).eval(bindings).asString()); + if (s == null) { + // Result of concatenation is null if any of the Values is null. + // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. + return ExprEval.of(null); + } else { builder.append(s); } } @@ -943,9 +956,12 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) throw new IAE("Function[%s] needs 2 arguments", name()); } - final String haystack = Strings.nullToEmpty(args.get(0).eval(bindings).asString()); - final String needle = Strings.nullToEmpty(args.get(1).eval(bindings).asString()); + final String haystack = NullHandling.nullToEmptyIfNeeded(args.get(0).eval(bindings).asString()); + final String needle = NullHandling.nullToEmptyIfNeeded(args.get(1).eval(bindings).asString()); + if (haystack == null || needle == null) { + return ExprEval.of(null); + } return ExprEval.of(haystack.indexOf(needle)); } } @@ -982,7 +998,9 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(arg.substring(index)); } } else { - return ExprEval.of(null); + // If starting index of substring is greater then the length of string, the result will be a zero length string. + // e.g. 'select substring("abc", 4,5) as c;' will return an empty string + return ExprEval.of(NullHandling.defaultStringValue()); } } } @@ -1005,8 +1023,11 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) final String arg = args.get(0).eval(bindings).asString(); final String pattern = args.get(1).eval(bindings).asString(); final String replacement = args.get(2).eval(bindings).asString(); + if (arg == null) { + return ExprEval.of(NullHandling.defaultStringValue()); + } return ExprEval.of( - Strings.nullToEmpty(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement)) + arg.replace(NullHandling.nullToEmptyIfNeeded(pattern), NullHandling.nullToEmptyIfNeeded(replacement)) ); } } @@ -1027,7 +1048,10 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final String arg = args.get(0).eval(bindings).asString(); - return ExprEval.of(StringUtils.toLowerCase(Strings.nullToEmpty(arg))); + if (arg == null) { + return ExprEval.of(NullHandling.defaultStringValue()); + } + return ExprEval.of(StringUtils.toLowerCase(arg)); } } @@ -1047,7 +1071,50 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) } final String arg = args.get(0).eval(bindings).asString(); - return ExprEval.of(StringUtils.toUpperCase(Strings.nullToEmpty(arg))); + if (arg == null) { + return ExprEval.of(NullHandling.defaultStringValue()); + } + return ExprEval.of(StringUtils.toUpperCase(arg)); + } + } + + class IsNullFunc implements Function + { + @Override + public String name() + { + return "isnull"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final ExprEval expr = args.get(0).eval(bindings); + return ExprEval.of(expr.isNull(), ExprType.LONG); + } + } + + class IsNotNullFunc implements Function + { + @Override + public String name() + { + return "notnull"; + } + + @Override + public ExprEval apply(List args, Expr.ObjectBinding bindings) + { + if (args.size() != 1) { + throw new IAE("Function[%s] needs 1 argument", name()); + } + + final ExprEval expr = args.get(0).eval(bindings); + return ExprEval.of(!expr.isNull(), ExprType.LONG); } } } diff --git a/common/src/test/java/io/druid/math/expr/EvalTest.java b/common/src/test/java/io/druid/math/expr/EvalTest.java index 75a06a03d6eb..c63a016b8203 100644 --- a/common/src/test/java/io/druid/math/expr/EvalTest.java +++ b/common/src/test/java/io/druid/math/expr/EvalTest.java @@ -20,6 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -139,9 +140,13 @@ public void testLongEval() Assert.assertEquals(1271055781L, evalLong("unix_timestamp('2010-04-12T07:03:01')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01+09:00')", bindings)); Assert.assertEquals(1271023381L, evalLong("unix_timestamp('2010-04-12T07:03:01.419+09:00')", bindings)); - - Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals("NULL", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + } else { + Assert.assertEquals("", eval("nvl(if(x == 9223372036854775807, '', 'x'), 'NULL')", bindings).asString()); + } Assert.assertEquals("x", eval("nvl(if(x == 9223372036854775806, '', 'x'), 'NULL')", bindings).asString()); + } @Test diff --git a/common/src/test/java/io/druid/math/expr/FunctionTest.java b/common/src/test/java/io/druid/math/expr/FunctionTest.java index 2351dcf16b4a..99c153da550c 100644 --- a/common/src/test/java/io/druid/math/expr/FunctionTest.java +++ b/common/src/test/java/io/druid/math/expr/FunctionTest.java @@ -20,6 +20,7 @@ package io.druid.math.expr; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -55,7 +56,12 @@ public void testCaseSearched() public void testConcat() { assertExpr("concat(x,' ',y)", "foo 2"); - assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); + if (NullHandling.useDefaultValuesForNull()) { + assertExpr("concat(x,' ',nonexistent,' ',y)", "foo 2"); + } else { + assertExpr("concat(x,' ',nonexistent,' ',y)", null); + } + assertExpr("concat(z)", "3.1"); assertExpr("concat()", null); } @@ -110,4 +116,18 @@ private void assertExpr(final String expression, final Object expectedResult) final Expr expr = Parser.parse(expression, ExprMacroTable.nil()); Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value()); } + + @Test + public void testIsNull() + { + assertExpr("isnull(null)", 1L); + assertExpr("isnull('abc')", 0L); + } + + @Test + public void testIsNotNull() + { + assertExpr("notnull(null)", 0L); + assertExpr("notnull('abc')", 1L); + } } diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 0903209b1f95..3c66cac3860b 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -19,16 +19,25 @@ package io.druid.query.aggregation.distinctcount; +import com.google.common.base.Preconditions; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop. + */ public class DistinctCountAggregator implements Aggregator { - + private static int UNKNOWN = -1; private final DimensionSelector selector; private final MutableBitmap mutableBitmap; + private int idForNull; public DistinctCountAggregator( DimensionSelector selector, @@ -37,6 +46,12 @@ public DistinctCountAggregator( { this.selector = selector; this.mutableBitmap = mutableBitmap; + Preconditions.checkArgument( + selector.nameLookupPossibleInAdvance() + || selector.getValueCardinality() != DimensionSelector.CARDINALITY_UNKNOWN, + "DistinctCountAggregator not supported for selector" + ); + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN; } @Override @@ -45,8 +60,24 @@ public void aggregate() IndexedInts row = selector.getRow(); for (int i = 0; i < row.size(); i++) { int index = row.get(i); - mutableBitmap.add(index); + if (NullHandling.useDefaultValuesForNull() || isNotNull(index)) { + mutableBitmap.add(index); + } + } + } + + private boolean isNotNull(int index) + { + if (idForNull == UNKNOWN) { + String value = selector.lookupName(index); + if (value == null) { + idForNull = index; + return false; + } else { + return true; + } } + return index != idForNull; } @Override diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index 389e2c4187cf..5f00dca9fe2c 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -19,8 +19,10 @@ package io.druid.query.aggregation.distinctcount; +import com.google.common.base.Preconditions; import io.druid.collections.bitmap.MutableBitmap; import io.druid.collections.bitmap.WrappedRoaringBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.aggregation.BufferAggregator; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.DimensionSelector; @@ -30,16 +32,31 @@ import java.nio.ByteBuffer; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop. + */ public class DistinctCountBufferAggregator implements BufferAggregator { + private static int UNKNOWN = -1; private final DimensionSelector selector; private final Int2ObjectMap mutableBitmapCollection = new Int2ObjectOpenHashMap<>(); + private int idForNull; + public DistinctCountBufferAggregator( DimensionSelector selector ) { this.selector = selector; + Preconditions.checkArgument( + selector.nameLookupPossibleInAdvance() + || selector.getValueCardinality() != DimensionSelector.CARDINALITY_UNKNOWN, + "DistinctCountBufferAggregator not supported for selector" + ); + this.idForNull = selector.nameLookupPossibleInAdvance() ? selector.idLookup().lookupId(null) : UNKNOWN; } @Override @@ -55,7 +72,9 @@ public void aggregate(ByteBuffer buf, int position) IndexedInts row = selector.getRow(); for (int i = 0; i < row.size(); i++) { int index = row.get(i); - mutableBitmap.add(index); + if (NullHandling.useDefaultValuesForNull() || isNotNull(index)) { + mutableBitmap.add(index); + } } buf.putLong(position, mutableBitmap.size()); } @@ -70,6 +89,20 @@ private MutableBitmap getMutableBitmap(int position) return mutableBitmap; } + private boolean isNotNull(int index) + { + if (idForNull == UNKNOWN) { + String value = selector.lookupName(index); + if (value == null) { + idForNull = index; + return false; + } else { + return true; + } + } + return index != idForNull; + } + @Override public Object get(ByteBuffer buf, int position) { diff --git a/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java b/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java index 5dfd1c176b1c..4c49c821520f 100644 --- a/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java +++ b/extensions-contrib/distinctcount/src/test/java/io/druid/query/aggregation/distinctcount/DistinctCountGroupByQueryTest.java @@ -35,23 +35,50 @@ import io.druid.query.groupby.GroupByQueryRunnerTestHelper; import io.druid.query.groupby.orderby.DefaultLimitSpec; import io.druid.query.groupby.orderby.OrderByColumnSpec; +import io.druid.query.groupby.strategy.GroupByStrategySelector; import io.druid.segment.IncrementalIndexSegment; +import io.druid.common.config.NullHandling; import io.druid.segment.Segment; import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.List; +@RunWith(Parameterized.class) public class DistinctCountGroupByQueryTest { + @Parameterized.Parameters(name = "{0}") + public static Collection constructorFeeder() throws IOException + { + final List constructors = Lists.newArrayList(); + for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { + constructors.add(new Object[]{config}); + } + + return constructors; + } + + private GroupByQueryConfig config; + + public DistinctCountGroupByQueryTest( + GroupByQueryConfig config + ) + { + this.config = config; + } @Test public void testGroupByWithDistinctCountAgg() throws Exception { - final GroupByQueryConfig config = new GroupByQueryConfig(); + // groupBy V2 skips adding null entries while doing serde. + boolean nullsInResult = config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V2); config.setMaxIntermediateRows(10000); final GroupByQueryRunnerFactory factory = GroupByQueryRunnerTest.makeQueryRunnerFactory(config); @@ -90,6 +117,36 @@ public void testGroupByWithDistinctCountAgg() throws Exception ImmutableMap.of(visitor_id, "2", client_type, "android") ) ); + // Row with Null values + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of() + ) + ); + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of(visitor_id, "2") + ) + ); + // Row with empty values + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of(visitor_id, "", client_type, "") + ) + ); + index.add( + new MapBasedInputRow( + timestamp + 2, + Lists.newArrayList(visitor_id, client_type), + ImmutableMap.of(visitor_id, "4", client_type, "") + ) + ); GroupByQuery query = new GroupByQuery.Builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -128,20 +185,70 @@ public void testGroupByWithDistinctCountAgg() throws Exception query ); - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "1970-01-01T00:00:00.000Z", - client_type, "iphone", - "UV", 2L, - "rows", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "1970-01-01T00:00:00.000Z", - client_type, "android", - "UV", 1L, - "rows", 1L - ) - ); + List expectedResults; + if (NullHandling.useDefaultValuesForNull()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "iphone", + "UV", 2L, + "rows", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "android", + "UV", 1L, + "rows", 1L + ), + // Both rows with null and empty string gets rolled up so rowCount is 3 + nullsInResult ? + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "UV", 3L, + "rows", 3L + ) : + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "client_type", null, + "UV", 3L, + "rows", 3L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "iphone", + "UV", 2L, + "rows", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "android", + "UV", 1L, + "rows", 1L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + client_type, "", + "UV", 2L, + "rows", 2L + ), + // client_type is null is this row + nullsInResult ? + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "UV", 1L, + "rows", 2L + ) : + GroupByQueryRunnerTestHelper.createExpectedRow( + "1970-01-01T00:00:00.000Z", + "client_type", null, + "UV", 1L, + "rows", 2L + ) + ); + } TestHelper.assertExpectedObjects(expectedResults, results, "distinct-count"); } } diff --git a/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java b/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java index f33d1d4a7f02..46afdc470372 100644 --- a/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java +++ b/extensions-contrib/time-min-max/src/main/java/io/druid/query/aggregation/TimestampAggregatorFactory.java @@ -29,6 +29,7 @@ import io.druid.segment.ObjectColumnSelector; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.sql.Timestamp; import java.util.Arrays; @@ -83,8 +84,15 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TimestampAggregator.combineValues(comparator, lhs, rhs); } @@ -159,9 +167,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return DateTimes.utc((long) object); + return object == null ? null : DateTimes.utc((long) object); } @Override diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index 98ea69b61d10..b9c9f477b13f 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -192,6 +192,12 @@ public long getLong() return 0L; } + @Override + public boolean isNull() + { + return false; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 8fae927cef8b..0cb2608b0a8e 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.granularity.Granularities; import io.druid.query.Druids; import io.druid.query.QueryDataSource; @@ -308,9 +309,14 @@ public void testQuantileOnInnerQuery() throws Exception // Verify results final List results = plannerResult.run().toList(); - final List expectedResults = ImmutableList.of( - new Object[]{7.0, 8.26386833190918} - ); + final List expectedResults = + NullHandling.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{7.0, 8.26386833190918} + ) : + ImmutableList.of( + new Object[]{5.25, 6.59091854095459} + ); Assert.assertEquals(expectedResults.size(), results.size()); for (int i = 0; i < expectedResults.size(); i++) { Assert.assertArrayEquals(expectedResults.get(i), results.get(i)); diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java index ff0a93c4da2c..c75b345d1358 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/query/lookup/namespace/UriExtractionNamespace.java @@ -396,8 +396,10 @@ public TSVFlatDataParser( "Must specify more than one column to have a key value pair" ); final DelimitedParser delegate = new DelimitedParser( + //CHECKSTYLE.OFF: Regexp Strings.emptyToNull(delimiter), Strings.emptyToNull(listDelimiter), + //CHECKSTYLE.ON: Regexp hasHeaderRow, skipHeaderRows ); diff --git a/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java b/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java index 08cc4ee6ee20..8635110ae161 100644 --- a/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java +++ b/extensions-core/lookups-cached-global/src/test/java/io/druid/server/lookup/namespace/cache/JdbcExtractionNamespaceTest.java @@ -384,7 +384,9 @@ public void testMappingWithoutFilter() String key = e.getKey(); String[] val = e.getValue(); String field = val[0]; + //CHECKSTYLE.OFF: Regexp Assert.assertEquals("non-null check", Strings.emptyToNull(field), Strings.emptyToNull(map.get(key))); + //CHECKSTYLE.ON: Regexp } Assert.assertEquals("null check", null, map.get("baz")); } @@ -415,9 +417,11 @@ public void testMappingWithFilter() String filterVal = val[1]; if (filterVal.equals("1")) { + //CHECKSTYLE.OFF: Regexp Assert.assertEquals("non-null check", Strings.emptyToNull(field), Strings.emptyToNull(map.get(key))); } else { Assert.assertEquals("non-null check", null, Strings.emptyToNull(map.get(key))); + //CHECKSTYLE.ON: Regexp } } } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java index 3534b322db3a..7f1b64310e61 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/LoadingLookup.java @@ -21,8 +21,8 @@ import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.logger.Logger; import io.druid.query.lookup.LookupExtractor; import io.druid.server.lookup.cache.loading.LoadingCache; @@ -71,7 +71,7 @@ public String apply(final String key) final String presentVal; try { presentVal = loadingCache.get(key, new ApplyCallable(key)); - return Strings.emptyToNull(presentVal); + return NullHandling.emptyToNullIfNeeded(presentVal); } catch (ExecutionException e) { LOGGER.debug("value not found for key [%s]", key); @@ -133,7 +133,7 @@ public ApplyCallable(String key) public String call() throws Exception { // avoid returning null and return an empty string to cache it. - return Strings.nullToEmpty(dataFetcher.fetch(key)); + return NullHandling.nullToEmptyIfNeeded(dataFetcher.fetch(key)); } } diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java index 0abc23b8e226..3a09ae9b248a 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/PollingLookup.java @@ -20,11 +20,11 @@ package io.druid.server.lookup; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; @@ -119,7 +119,7 @@ public String apply(@NotNull String key) // it must've been closed after swapping while I was getting it. Try again. return this.apply(key); } - return Strings.emptyToNull((String) cache.get(key)); + return NullHandling.emptyToNullIfNeeded((String) cache.get(key)); } finally { if (cacheRefKeeper != null && cache != null) { diff --git a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java index 6d0af4e821fb..e1ac561d4ff1 100644 --- a/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java +++ b/extensions-core/lookups-cached-single/src/main/java/io/druid/server/lookup/jdbc/JdbcDataFetcher.java @@ -21,9 +21,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.metadata.MetadataStorageConnectorConfig; @@ -132,7 +132,7 @@ public List inTransaction(Handle handle, TransactionStatus status) throw if (pairs.isEmpty()) { return null; } - return Strings.nullToEmpty(pairs.get(0)); + return NullHandling.nullToEmptyIfNeeded(pairs.get(0)); } @Override diff --git a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java index dee07f60332e..4afaed01feae 100644 --- a/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java +++ b/extensions-core/lookups-cached-single/src/test/java/io/druid/server/lookup/PollingLookupTest.java @@ -21,11 +21,11 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.ISE; import io.druid.query.lookup.LookupExtractor; import io.druid.server.lookup.cache.polling.OffHeapPollingCache; @@ -191,7 +191,7 @@ public void testBulkApply() public String apply(String input) { //make sure to rewrite null strings as empty. - return Strings.nullToEmpty(input); + return NullHandling.nullToEmptyIfNeeded(input); } })); } @@ -208,7 +208,7 @@ private void assertMapLookup(Map map, LookupExtractor lookup) for (Map.Entry entry : map.entrySet()) { String key = entry.getKey(); String val = entry.getValue(); - Assert.assertEquals("non-null check", Strings.emptyToNull(val), lookup.apply(key)); + Assert.assertEquals("non-null check", NullHandling.emptyToNullIfNeeded(val), lookup.apply(key)); } } } diff --git a/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java b/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java index ec20b5e37fe3..3448dd4334ea 100644 --- a/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java +++ b/extensions-core/stats/src/test/java/io/druid/query/aggregation/variance/VarianceAggregatorCollectorTest.java @@ -151,6 +151,13 @@ public float getFloat() { return v; } + + @Override + public boolean isNull() + { + return false; + } + } private static class ObjectHandOver extends TestObjectColumnSelector diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index c9111d00cd35..a4e6f9a35032 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -73,6 +73,7 @@ import org.joda.time.DateTime; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -419,6 +420,7 @@ public List getDimension(String dimension) } @Override + @Nullable public Object getRaw(String dimension) { return row.getRaw(dimension); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 672f77133746..e3f85992b483 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -38,6 +38,7 @@ import io.druid.segment.serde.ComplexMetrics; import org.apache.hadoop.io.WritableUtils; +import javax.annotation.Nullable; import java.io.DataInput; import java.io.IOException; import java.util.List; @@ -48,6 +49,8 @@ public class InputRowSerde { private static final Logger log = new Logger(InputRowSerde.class); + private static final byte NULL_BYTE = (byte) 1; + private static final byte NON_NULL_BYTE = (byte) 0; public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs, boolean reportParseExceptions) { @@ -103,18 +106,22 @@ public InputRow get() } String t = aggFactory.getTypeName(); - - if (t.equals("float")) { - out.writeFloat(agg.getFloat()); - } else if (t.equals("long")) { - WritableUtils.writeVLong(out, agg.getLong()); - } else if (t.equals("double")) { - out.writeDouble(agg.getDouble()); + if (agg.isNull()) { + out.writeByte(NULL_BYTE); } else { - //its a complex metric - Object val = agg.get(); - ComplexMetricSerde serde = getComplexMetricSerde(t); - writeBytes(serde.toBytes(val), out); + out.writeByte(NON_NULL_BYTE); + if (t.equals("float")) { + out.writeFloat(agg.getFloat()); + } else if (t.equals("long")) { + WritableUtils.writeVLong(out, agg.getLong()); + } else if (t.equals("double")) { + out.writeDouble(agg.getDouble()); + } else { + //its a complex metric + Object val = agg.get(); + ComplexMetricSerde serde = getComplexMetricSerde(t); + writeBytes(serde.toBytes(val), out); + } } } } @@ -126,15 +133,18 @@ public InputRow get() } } - private static void writeBytes(byte[] value, ByteArrayDataOutput out) throws IOException + private static void writeBytes(@Nullable byte[] value, ByteArrayDataOutput out) throws IOException { - WritableUtils.writeVInt(out, value.length); - out.write(value, 0, value.length); + int length = value == null ? -1 : value.length; + WritableUtils.writeVInt(out, length); + if (value != null) { + out.write(value, 0, value.length); + } } - private static void writeString(String value, ByteArrayDataOutput out) throws IOException + private static void writeString(@Nullable String value, ByteArrayDataOutput out) throws IOException { - writeBytes(StringUtils.toUtf8(value), out); + writeBytes(StringUtils.toUtf8Nullable(value), out); } private static void writeStringArray(List values, ByteArrayDataOutput out) throws IOException @@ -149,15 +159,20 @@ private static void writeStringArray(List values, ByteArrayDataOutput ou } } + @Nullable private static String readString(DataInput in) throws IOException { byte[] result = readBytes(in); - return StringUtils.fromUtf8(result); + return StringUtils.fromUtf8Nullable(result); } + @Nullable private static byte[] readBytes(DataInput in) throws IOException { int size = WritableUtils.readVInt(in); + if (size < 0) { + return null; + } byte[] result = new byte[size]; in.readFully(result, 0, size); return result; @@ -208,6 +223,11 @@ public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) for (int i = 0; i < metricSize; i++) { String metric = readString(in); String type = getType(metric, aggs, i); + byte metricNullability = in.readByte(); + if (metricNullability == NULL_BYTE) { + // metric value is null. + continue; + } if (type.equals("float")) { event.put(metric, in.readFloat()); } else if (type.equals("long")) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 16809561cbbc..be2aa2bbda9b 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -302,8 +302,10 @@ public static void injectSystemProperties(Job job) public static void injectDruidProperties(Configuration configuration, List listOfAllowedPrefix) { + //CHECKSTYLE.OFF: Regexp String mapJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.MAP_JAVA_OPTS)); String reduceJavaOpts = Strings.nullToEmpty(configuration.get(MRJobConfig.REDUCE_JAVA_OPTS)); + //CHECKSTYLE.ON: Regexp for (String propName : System.getProperties().stringPropertyNames()) { for (String prefix : listOfAllowedPrefix) { diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java index cf077eb32b2c..fae91a2c59c8 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/SegmentInputRow.java @@ -23,6 +23,7 @@ import io.druid.data.input.Row; import org.joda.time.DateTime; +import javax.annotation.Nullable; import java.util.List; /** @@ -67,6 +68,7 @@ public List getDimension(String dimension) } @Override + @Nullable public Object getRaw(String dimension) { return delegate.getRaw(dimension); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java index 37519fd67e4b..82e305157ff8 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/path/StaticPathSpec.java @@ -128,7 +128,9 @@ public static void addToMultipleInputs( private static void addInputPath(Job job, Iterable pathStrings, Class inputFormatClass) { Configuration conf = job.getConfiguration(); + //CHECKSTYLE.OFF: Regexp StringBuilder inputFormats = new StringBuilder(Strings.nullToEmpty(conf.get(MultipleInputs.DIR_FORMATS))); + //CHECKSTYLE.ON: Regexp String[] paths = Iterables.toArray(pathStrings, String.class); for (int i = 0; i < paths.length - 1; i++) { diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index 67a9b5abf8a3..daad96fca838 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -33,6 +33,7 @@ import io.druid.query.aggregation.LongSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.segment.ColumnSelectorFactory; +import io.druid.common.config.NullHandling; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -71,6 +72,7 @@ public void testSerde() { // Prepare the mocks & set close() call count expectation to 1 final Aggregator mockedAggregator = EasyMock.createMock(DoubleSumAggregator.class); + EasyMock.expect(mockedAggregator.isNull()).andReturn(false).times(1); EasyMock.expect(mockedAggregator.getDouble()).andReturn(0d).times(1); mockedAggregator.aggregate(); EasyMock.expectLastCall().times(1); @@ -78,6 +80,26 @@ public void testSerde() EasyMock.expectLastCall().times(1); EasyMock.replay(mockedAggregator); + final Aggregator mockedNullAggregator = EasyMock.createMock(DoubleSumAggregator.class); + EasyMock.expect(mockedNullAggregator.isNull()).andReturn(true).times(1); + mockedNullAggregator.aggregate(); + EasyMock.expectLastCall().times(1); + mockedNullAggregator.close(); + EasyMock.expectLastCall().times(1); + EasyMock.replay(mockedNullAggregator); + + final AggregatorFactory mockedAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); + EasyMock.expect(mockedAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedAggregator); + EasyMock.expect(mockedAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + EasyMock.expect(mockedAggregatorFactory.getName()).andReturn("mockedAggregator").anyTimes(); + + final AggregatorFactory mockedNullAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); + EasyMock.expect(mockedNullAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedNullAggregator); + EasyMock.expect(mockedNullAggregatorFactory.getName()).andReturn("mockedNullAggregator").anyTimes(); + EasyMock.expect(mockedNullAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + + EasyMock.replay(mockedAggregatorFactory, mockedNullAggregatorFactory); + InputRow in = new MapBasedInputRow( timestamp, dims, @@ -90,13 +112,8 @@ public void testSerde() new LongSumAggregatorFactory("m2out", "m2"), new HyperUniquesAggregatorFactory("m3out", "m3"), new LongSumAggregatorFactory("unparseable", "m3"), // Unparseable from String to Long - new DoubleSumAggregatorFactory("mockedAggregator", "m4") { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - return mockedAggregator; - } - } + mockedAggregatorFactory, + mockedNullAggregatorFactory }; byte[] data = InputRowSerde.toBytes(in, aggregatorFactories, false); // Ignore Unparseable aggregator @@ -108,13 +125,19 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) Assert.assertEquals(ImmutableList.of("d1v"), out.getDimension("d1")); Assert.assertEquals(ImmutableList.of("d2v1", "d2v2"), out.getDimension("d2")); - Assert.assertEquals(0.0f, out.getMetric("agg_non_existing").floatValue(), 0.00001); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(0.0f, out.getMetric("agg_non_existing").floatValue(), 0.00001); + } else { + Assert.assertNull(out.getMetric("agg_non_existing")); + } + + Assert.assertEquals(0L, out.getMetric("unparseable")); Assert.assertEquals(5.0f, out.getMetric("m1out").floatValue(), 0.00001); Assert.assertEquals(100L, out.getMetric("m2out")); Assert.assertEquals(1, ((HyperLogLogCollector) out.getRaw("m3out")).estimateCardinality(), 0.001); - Assert.assertEquals(0L, out.getMetric("unparseable")); EasyMock.verify(mockedAggregator); + EasyMock.verify(mockedNullAggregator); } @Test(expected = ParseException.class) diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index 5606219396f3..dc4f09fa6a0e 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -36,6 +36,7 @@ import com.metamx.metrics.MonitorScheduler; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; +import io.druid.common.config.NullHandling; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; @@ -119,6 +120,7 @@ import io.druid.server.coordination.DataSegmentServerAnnouncer; import io.druid.server.coordination.ServerType; import io.druid.timeline.DataSegment; +import io.druid.utils.Runnables; import org.easymock.EasyMock; import org.hamcrest.CoreMatchers; import org.joda.time.DateTime; @@ -207,7 +209,7 @@ public InputRow nextRow() @Override public Runnable commit() { - return () -> {}; + return Runnables.getNoopRunnable(); } @Override @@ -353,8 +355,8 @@ public void testBasics() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(2, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(2, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -422,10 +424,14 @@ public void testTransformSpec() throws Exception Assert.assertEquals(0, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(1, sumMetric(task, null, "rows")); - Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows")); - Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); - Assert.assertEquals(1, sumMetric(task, null, "met1")); + Assert.assertEquals(1, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(1, sumMetric(task, new SelectorDimFilter("dim1t", "foofoo", null), "rows").longValue()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(0, sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows").longValue()); + } else { + Assert.assertNull(sumMetric(task, new SelectorDimFilter("dim1t", "barbar", null), "rows")); + } + Assert.assertEquals(1, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -551,8 +557,8 @@ public void testNoReportParseExceptions() throws Exception Assert.assertEquals(2, task.getMetrics().unparseable()); // Do some queries. - Assert.assertEquals(3, sumMetric(task, null, "rows")); - Assert.assertEquals(3, sumMetric(task, null, "met1")); + Assert.assertEquals(3, sumMetric(task, null, "rows").longValue()); + Assert.assertEquals(3, sumMetric(task, null, "met1").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -624,7 +630,7 @@ public void testRestore() throws Exception } // Do a query, at this point the previous data should be loaded. - Assert.assertEquals(1, sumMetric(task2, null, "rows")); + Assert.assertEquals(1, sumMetric(task2, null, "rows").longValue()); final TestFirehose firehose = (TestFirehose) task2.getFirehose(); @@ -645,7 +651,7 @@ public void testRestore() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(2, sumMetric(task2, null, "rows")); + Assert.assertEquals(2, sumMetric(task2, null, "rows").longValue()); // Simulate handoff. for (Map.Entry> entry : handOffCallbacks.entrySet()) { @@ -706,7 +712,7 @@ public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception publishedSegment = Iterables.getOnlyElement(mdc.getPublished()); // Do a query. - Assert.assertEquals(1, sumMetric(task1, null, "rows")); + Assert.assertEquals(1, sumMetric(task1, null, "rows").longValue()); // Trigger graceful shutdown. task1.stopGracefully(); @@ -1095,7 +1101,7 @@ public List getLocations() return toolboxFactory.build(task); } - public long sumMetric(final Task task, final DimFilter filter, final String metric) throws Exception + public Long sumMetric(final Task task, final DimFilter filter, final String metric) throws Exception { // Do a query. TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() @@ -1111,6 +1117,6 @@ public long sumMetric(final Task task, final DimFilter filter, final String metr List> results = task.getQueryRunner(query).run(QueryPlus.wrap(query), ImmutableMap.of()).toList(); - return results.isEmpty() ? 0 : results.get(0).getValue().getLongMetric(metric); + return results.isEmpty() ? NullHandling.defaultLongValue() : results.get(0).getValue().getLongMetric(metric); } } diff --git a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java index dbf84aad7315..0d7d84b1c544 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StringUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StringUtils.java @@ -22,6 +22,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; +import javax.annotation.Nullable; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -77,11 +78,37 @@ public static String fromUtf8(final byte[] bytes) } } + @Nullable + public static String fromUtf8Nullable(@Nullable final byte[] bytes) + { + if (bytes == null) { + return null; + } + return fromUtf8(bytes); + } + public static String fromUtf8(final ByteBuffer buffer, final int numBytes) { final byte[] bytes = new byte[numBytes]; buffer.get(bytes); - return StringUtils.fromUtf8(bytes); + return fromUtf8(bytes); + } + + /** + * Reads numBytes bytes from buffer and converts that to a utf-8 string + * @param buffer buffer to read bytes from + * @param numBytes number of bytes to read + * @return returns null if numBytes is -1 otherwise utf-8 string represetation of bytes read + */ + @Nullable + public static String fromUtf8Nullable(final ByteBuffer buffer, final int numBytes) + { + if (numBytes < 0) { + return null; + } + final byte[] bytes = new byte[numBytes]; + buffer.get(bytes); + return fromUtf8Nullable(bytes); } public static String fromUtf8(final ByteBuffer buffer) @@ -100,6 +127,15 @@ public static byte[] toUtf8(final String string) } } + @Nullable + public static byte[] toUtf8Nullable(@Nullable final String string) + { + if (string == null) { + return null; + } + return toUtf8(string); + } + /** * Equivalent of String.format(Locale.ENGLISH, message, formatArgs). */ diff --git a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java index e3975ca15d44..6e2d7a004c5e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/parsers/ParserUtils.java @@ -52,6 +52,7 @@ public class ParserUtils } } + //CHECKSTYLE.OFF: Regexp public static Function getMultiValueFunction( final String listDelimiter, final Splitter listSplitter @@ -67,6 +68,7 @@ public static Function getMultiValueFunction( } }; } + //CHECKSTYLE.ON: Regexp public static ArrayList generateFieldNames(int length) { diff --git a/pom.xml b/pom.xml index 41233b5e42cc..133ccee97c2d 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ 2.5.5 3.4.11 + true @@ -1039,6 +1040,7 @@ -Xmx3000m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 -Duser.timezone=UTC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -Ddruid.generic.useDefaultValueForNull=${druid.generic.useDefaultValueForNull} -Ddruid.indexing.doubleStorage=double diff --git a/processing/src/main/java/io/druid/guice/GuiceInjectors.java b/processing/src/main/java/io/druid/guice/GuiceInjectors.java index f3bac2362ce4..8adaae4dd25d 100644 --- a/processing/src/main/java/io/druid/guice/GuiceInjectors.java +++ b/processing/src/main/java/io/druid/guice/GuiceInjectors.java @@ -42,6 +42,7 @@ public static Collection makeDefaultStartupModules() new JacksonModule(), new PropertiesModule(Arrays.asList("common.runtime.properties", "runtime.properties")), new ConfigModule(), + new NullHandlingModule(), new Module() { @Override diff --git a/processing/src/main/java/io/druid/guice/NullHandlingModule.java b/processing/src/main/java/io/druid/guice/NullHandlingModule.java new file mode 100644 index 000000000000..7216ca363822 --- /dev/null +++ b/processing/src/main/java/io/druid/guice/NullHandlingModule.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.common.config.NullHandling; +import io.druid.common.config.NullValueHandlingConfig; + +/** + */ +public class NullHandlingModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.generic", NullValueHandlingConfig.class); + binder.requestStaticInjection(NullHandling.class); + binder.requestStaticInjection(NullHandling.class); + + } +} diff --git a/processing/src/main/java/io/druid/jackson/StringComparatorModule.java b/processing/src/main/java/io/druid/jackson/StringComparatorModule.java index 4ef8783a48e7..95ebfba45bad 100644 --- a/processing/src/main/java/io/druid/jackson/StringComparatorModule.java +++ b/processing/src/main/java/io/druid/jackson/StringComparatorModule.java @@ -41,5 +41,7 @@ public StringComparatorModule() } @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringComparator.class) - public interface StringComparatorMixin {} + public interface StringComparatorMixin + { + } } diff --git a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java index 7d830fedac00..0fbaeb5fdec8 100644 --- a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java @@ -118,7 +118,9 @@ public void duration(QueryType query) @Override public void queryId(QueryType query) { + //CHECKSTYLE.OFF: Regexp setDimension(DruidMetrics.ID, Strings.nullToEmpty(query.getId())); + //CHECKSTYLE.ON: Regexp } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java index 2287307eb749..797f44843837 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregateCombiner.java @@ -70,6 +70,12 @@ public interface AggregateCombiner extends ColumnValueSelector @SuppressWarnings("unused") // Going to be used when https://github.com/druid-io/druid/projects/2 is complete void fold(ColumnValueSelector selector); + @Override + default boolean isNull() + { + return false; + } + @Override default void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java index a13eebf1b371..a131351eef89 100644 --- a/processing/src/main/java/io/druid/query/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -28,7 +28,7 @@ * An Aggregator is an object that can aggregate metrics. Its aggregation-related methods (namely, aggregate() and get()) * do not take any arguments as the assumption is that the Aggregator was given something in its constructor that * it can use to get at the next bit of data. - * + *

* Thus, an Aggregator can be thought of as a closure over some other thing that is stateful and changes between calls * to aggregate(). This is currently (as of this documentation) implemented through the use of {@link * io.druid.segment.ColumnValueSelector} objects. @@ -40,7 +40,9 @@ public interface Aggregator extends Closeable @Nullable Object get(); + float getFloat(); + long getLong(); /** @@ -53,6 +55,19 @@ default double getDouble() return (double) getFloat(); } + /** + * returns true if aggregator's output type is primitive long/double/float and aggregated value is null, + * but when aggregated output type is Object, this method always returns false, + * and users are advised to check nullability for the object returned by {@link #get()} + * method. + * The default implementation always return false to enable smooth backward compatibility, + * re-implement if your aggregator is nullable. + */ + default boolean isNull() + { + return false; + } + @Override void close(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index d5e6790c9ab9..cadde722e50b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -36,6 +36,8 @@ * AggregatorFactory is a strategy (in the terms of Design Patterns) that represents column aggregation, e. g. min, * max, sum of metric columns, or cardinality of dimension columns (see {@link * io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory}). + * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * to extend {@link io.druid.query.aggregation.NullableAggregatorFactory}. */ @ExtensionPoint public abstract class AggregatorFactory implements Cacheable @@ -59,7 +61,8 @@ public abstract class AggregatorFactory implements Cacheable * * @return an object representing the combination of lhs and rhs, this can be a new object or a mutation of the inputs */ - public abstract Object combine(Object lhs, Object rhs); + @Nullable + public abstract Object combine(@Nullable Object lhs, @Nullable Object rhs); /** * Creates an AggregateCombiner to fold rollup aggregation results from serveral "rows" of different indexes during @@ -127,7 +130,8 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * * @return the finalized value that should be returned for the initial query */ - public abstract Object finalizeComputation(Object object); + @Nullable + public abstract Object finalizeComputation(@Nullable Object object); public abstract String getName(); diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 7f3591182605..f1b2438fe02b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -37,6 +37,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.virtual.ExpressionSelectors; +import javax.annotation.Nullable; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -141,7 +142,7 @@ public static BaseFloatColumnValueSelector makeColumnValueSelectorWithFloatDefau final ExprMacroTable macroTable, final String fieldName, final String fieldExpression, - final float nullValue + @Nullable final Float nullValue ) { if (fieldName != null && fieldExpression == null) { @@ -164,6 +165,13 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("baseSelector", baseSelector); } + + @Override + public boolean isNull() + { + final ExprEval exprEval = baseSelector.getObject(); + return exprEval.isNull(); + } } return new ExpressionFloatColumnSelector(); } @@ -198,6 +206,13 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("baseSelector", baseSelector); } + + @Override + public boolean isNull() + { + final ExprEval exprEval = baseSelector.getObject(); + return exprEval.isNull(); + } } return new ExpressionLongColumnSelector(); } @@ -232,6 +247,13 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("baseSelector", baseSelector); } + + @Override + public boolean isNull() + { + final ExprEval exprEval = baseSelector.getObject(); + return exprEval.isNull(); + } } return new ExpressionDoubleColumnSelector(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index cdddd7629f2d..6385cb3405b7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -30,7 +30,7 @@ * A BufferAggregator is an object that can aggregate metrics into a ByteBuffer. Its aggregation-related methods * (namely, aggregate(...) and get(...)) only take the ByteBuffer and position because it is assumed that the Aggregator * was given something (one or more MetricSelector(s)) in its constructor that it can use to get at the next bit of data. - * + *

* Thus, an Aggregator can be thought of as a closure over some other thing that is stateful and changes between calls * to aggregate(...). */ @@ -39,15 +39,15 @@ public interface BufferAggregator extends HotLoopCallee { /** * Initializes the buffer location - * + *

* Implementations of this method must initialize the byte buffer at the given position - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* This method must not exceed the number of bytes returned by {@link AggregatorFactory#getMaxIntermediateSize()} * in the corresponding {@link AggregatorFactory} * - * @param buf byte buffer to initialize + * @param buf byte buffer to initialize * @param position offset within the byte buffer for initialization */ @CalledFromHotLoop @@ -55,13 +55,13 @@ public interface BufferAggregator extends HotLoopCallee /** * Aggregates metric values into the given aggregate byte representation - * + *

* Implementations of this method must read in the aggregate value from the buffer at the given position, * aggregate the next element of data and write the updated aggregate value back into the buffer. - * + *

* Implementations must not change the position, limit or mark of the given buffer * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the current aggregate value is stored */ @CalledFromHotLoop @@ -69,68 +69,72 @@ public interface BufferAggregator extends HotLoopCallee /** * Returns the intermediate object representation of the given aggregate. - * + *

* Converts the given byte buffer representation into an intermediate aggregate Object - * + *

* Implementations must not change the position, limit or mark of the given buffer * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the Object representation of the aggregate */ Object get(ByteBuffer buf, int position); /** * Returns the float representation of the given aggregate byte array - * + *

* Converts the given byte buffer representation into the intermediate aggregate value. - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* Implementations are only required to support this method if they are aggregations which * have an {@link AggregatorFactory#getTypeName()} of "float". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the float representation of the aggregate */ float getFloat(ByteBuffer buf, int position); /** * Returns the long representation of the given aggregate byte array - * + *

* Converts the given byte buffer representation into the intermediate aggregate value. - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* Implementations are only required to support this method if they are aggregations which * have an {@link AggregatorFactory#getTypeName()} of "long". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the long representation of the aggregate */ long getLong(ByteBuffer buf, int position); /** * Returns the double representation of the given aggregate byte array - * + *

* Converts the given byte buffer representation into the intermediate aggregate value. - * + *

* Implementations must not change the position, limit or mark of the given buffer - * + *

* Implementations are only required to support this method if they are aggregations which * have an {@link AggregatorFactory#getTypeName()} of "double". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. - * + *

* The default implementation casts {@link BufferAggregator#getFloat(ByteBuffer, int)} to double. * This default method is added to enable smooth backward compatibility, please re-implement it if your aggregators * work with numeric double columns. * - * @param buf byte buffer storing the byte array representation of the aggregate + * @param buf byte buffer storing the byte array representation of the aggregate * @param position offset within the byte buffer at which the aggregate value is stored + * * @return the double representation of the aggregate */ default double getDouble(ByteBuffer buf, int position) @@ -145,7 +149,7 @@ default double getDouble(ByteBuffer buf, int position) /** * {@inheritDoc} - * + *

*

The default implementation inspects nothing. Classes that implement {@code BufferAggregator} are encouraged to * override this method, following the specification of {@link HotLoopCallee#inspectRuntimeShape}. */ @@ -161,21 +165,40 @@ default void inspectRuntimeShape(RuntimeShapeInspector inspector) * built on top of old ByteBuffer can not be used for further {@link BufferAggregator#aggregate(ByteBuffer, int)} * calls. This method tells the BufferAggregator that the cached objects at a certain location has been relocated to * a different location. - * + *

* Only used if there is any positional caches/objects in the BufferAggregator implementation. - * + *

* If relocate happens to be across multiple new ByteBuffers (say n ByteBuffers), this method should be called * multiple times(n times) given all the new positions/old positions should exist in newBuffer/OldBuffer. - * + *

* Implementations must not change the position, limit or mark of the given buffer * * @param oldPosition old position of a cached object before aggregation buffer relocates to a new ByteBuffer. * @param newPosition new position of a cached object after aggregation buffer relocates to a new ByteBuffer. - * @param oldBuffer old aggregation buffer. - * @param newBuffer new aggregation buffer. + * @param oldBuffer old aggregation buffer. + * @param newBuffer new aggregation buffer. */ default void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) { } + /** + * returns true if aggregator's output type is primitive long/double/float and aggregated value is null, + * but when aggregated output type is Object, this method always returns false, + * and users are advised to check nullability for the object returned by {@link #get()} + * method. + * The default implementation always return false to enable smooth backward compatibility, + * re-implement if your aggregator is nullable. + * + * @param buf byte buffer storing the byte array representation of the aggregate + * @param position offset within the byte buffer at which the aggregate value is stored + * + * @return true if the aggrgeated value is null otherwise false. + * For backwards compatibility, isNull() may return false even if {@link BufferAggregator#get(ByteBuffer, int)} returns null. Users of this method should account for this case. + */ + default boolean isNull(ByteBuffer buf, int position) + { + return false; + } + } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 676647a21767..4d02de7cbec6 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -34,7 +38,6 @@ */ public class DoubleMaxAggregatorFactory extends SimpleDoubleAggregatorFactory { - @JsonCreator public DoubleMaxAggregatorFactory( @JsonProperty("name") String name, @@ -52,25 +55,46 @@ public DoubleMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleMaxAggregator(getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.NEGATIVE_INFINITY + ); + return Pair.of( + new DoubleMaxAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleMaxBufferAggregator(getDoubleColumnSelector(metricFactory, Double.NEGATIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.NEGATIVE_INFINITY + ); + return Pair.of( + new DoubleMaxBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleMaxAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMaxAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index eb480a6397d6..7869bb1f2ce3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -51,25 +55,46 @@ public DoubleMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleMinAggregator(getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.POSITIVE_INFINITY + ); + return Pair.of( + new DoubleMinAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleMinBufferAggregator(getDoubleColumnSelector(metricFactory, Double.POSITIVE_INFINITY)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector( + metricFactory, + Double.POSITIVE_INFINITY + ); + return Pair.of( + new DoubleMinBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleMinAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMinAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 34e398258d38..313a2676fdba 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -52,25 +56,40 @@ public DoubleSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleSumAggregator(getDoubleColumnSelector(metricFactory, 0.0)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); + return Pair.of( + new DoubleSumAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(getDoubleColumnSelector(metricFactory, 0.0)); + BaseDoubleColumnValueSelector doubleColumnSelector = getDoubleColumnSelector(metricFactory, 0.0); + return Pair.of( + new DoubleSumBufferAggregator(doubleColumnSelector), + doubleColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleSumAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleSumAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java index ddd164cf4093..589c1def93e7 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -51,25 +55,43 @@ public FloatMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatMaxAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.NEGATIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.NEGATIVE_INFINITY + ); + return Pair.of(new FloatMaxAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatMaxBufferAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.NEGATIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.NEGATIVE_INFINITY + ); + return Pair.of( + new FloatMaxBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatMaxAggregator.combineValues(finalizeComputation(lhs), finalizeComputation(rhs)); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMaxAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java index 17c078c6cfa3..f97d7c5ba29f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -51,25 +55,43 @@ public FloatMinAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatMinAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.POSITIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.POSITIVE_INFINITY + ); + return Pair.of(new FloatMinAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatMinBufferAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, Float.POSITIVE_INFINITY)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault( + metricFactory, + Float.POSITIVE_INFINITY + ); + return Pair.of( + new FloatMinBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatMinAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleMinAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java index 2767a61d311f..b6c7725cf970 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -22,10 +22,14 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -51,25 +55,37 @@ public FloatSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatSumAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); + return Pair.of(new FloatSumAggregator(floatColumnSelector), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatSumBufferAggregator(makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f)); + BaseFloatColumnValueSelector floatColumnSelector = makeColumnValueSelectorWithFloatDefault(metricFactory, 0.0f); + return Pair.of( + new FloatSumBufferAggregator(floatColumnSelector), + floatColumnSelector + ); } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatSumAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new DoubleSumAggregateCombiner(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index a63c93cb5b72..9487c5000354 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -24,13 +24,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -40,7 +43,7 @@ /** */ -public class LongMaxAggregatorFactory extends AggregatorFactory +public class LongMaxAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -73,15 +76,20 @@ public LongMaxAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongMaxAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of(new LongMaxAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongMaxBufferAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of( + new LongMaxBufferAggregator(longColumnSelector), + longColumnSelector + ); } private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -102,13 +110,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return LongMaxAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new LongAggregateCombiner() { @@ -163,7 +178,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } @@ -216,7 +232,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Longs.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index fc5890cc54e2..7ea18903e91e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -22,15 +22,19 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -40,7 +44,7 @@ /** */ -public class LongMinAggregatorFactory extends AggregatorFactory +public class LongMinAggregatorFactory extends NullableAggregatorFactory { private final String name; @@ -68,21 +72,27 @@ public LongMinAggregatorFactory( this.macroTable = macroTable; } + @VisibleForTesting public LongMinAggregatorFactory(String name, String fieldName) { this(name, fieldName, null, ExprMacroTable.nil()); } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongMinAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of(new LongMinAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongMinBufferAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of( + new LongMinBufferAggregator(longColumnSelector), + longColumnSelector + ); } private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -103,13 +113,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return LongMinAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new LongAggregateCombiner() { @@ -164,7 +181,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } @@ -217,7 +235,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Longs.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 3a2d07937614..22eda3a1296d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -24,12 +24,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.Parser; import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; @@ -39,7 +42,7 @@ /** */ -public class LongSumAggregatorFactory extends AggregatorFactory +public class LongSumAggregatorFactory extends NullableAggregatorFactory { private final String name; private final String fieldName; @@ -72,15 +75,20 @@ public LongSumAggregatorFactory(String name, String fieldName) } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongSumAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of(new LongSumAggregator(longColumnSelector), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(getLongColumnSelector(metricFactory)); + BaseLongColumnValueSelector longColumnSelector = getLongColumnSelector(metricFactory); + return Pair.of( + new LongSumBufferAggregator(longColumnSelector), + longColumnSelector + ); } private BaseLongColumnValueSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) @@ -103,11 +111,17 @@ public Comparator getComparator() @Override public Object combine(Object lhs, Object rhs) { + if (lhs == null) { + return rhs; + } + if (rhs == null) { + return lhs; + } return LongSumAggregator.combineValues(lhs, rhs); } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { return new LongSumAggregateCombiner(); } @@ -141,7 +155,8 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } @@ -194,7 +209,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Longs.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java new file mode 100644 index 000000000000..e33b7b458a4f --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregateCombiner.java @@ -0,0 +1,111 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; + +/** + * The result of a NullableAggregateCombiner will be null if all the values to be combined are null values or no values are combined at all. + * If any of the value is non-null, the result would be the value of the delegate combiner. + * Note that the delegate combiner is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate combiner. + */ +public class NullableAggregateCombiner implements AggregateCombiner +{ + private boolean isNullResult = true; + + private final AggregateCombiner delegate; + + public NullableAggregateCombiner(AggregateCombiner delegate) + { + this.delegate = delegate; + } + + @Override + public void reset(ColumnValueSelector selector) + { + if (selector.isNull()) { + isNullResult = true; + } else { + isNullResult = false; + delegate.reset(selector); + } + } + + @Override + public void fold(ColumnValueSelector selector) + { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (isNullResult) { + isNullResult = false; + } + delegate.fold(selector); + } + } + + @Override + public float getFloat() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return primitive float for Null Value"); + } + return delegate.getFloat(); + } + + @Override + public double getDouble() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return double for Null Value"); + } + return delegate.getDouble(); + } + + @Override + public long getLong() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return long for Null Value"); + } + return delegate.getLong(); + } + + @Override + public boolean isNull() + { + return isNullResult || delegate.isNull(); + } + + @Nullable + @Override + public Object getObject() + { + return isNullResult ? null : delegate.getObject(); + } + + @Override + public Class classOfObject() + { + return delegate.classOfObject(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java new file mode 100644 index 000000000000..1eac57edc34b --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregator.java @@ -0,0 +1,104 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.BaseNullableColumnValueSelector; + +import javax.annotation.Nullable; + +/** + * The result of a NullableAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. + * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. + * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate aggregator. + */ +public class NullableAggregator implements Aggregator +{ + private final Aggregator delegate; + private final BaseNullableColumnValueSelector selector; + private boolean isNullResult = true; + + public NullableAggregator(Aggregator delegate, BaseNullableColumnValueSelector selector) + { + this.delegate = delegate; + this.selector = selector; + } + + @Override + public void aggregate() + { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (isNullResult) { + isNullResult = false; + } + delegate.aggregate(); + } + } + + @Override + @Nullable + public Object get() + { + if (isNullResult) { + return null; + } + return delegate.get(); + } + + @Override + public float getFloat() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return float for Null Value"); + } + return delegate.getFloat(); + } + + @Override + public long getLong() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return long for Null Value"); + } + return delegate.getLong(); + } + + @Override + public double getDouble() + { + if (isNullResult) { + throw new IllegalStateException("Cannot return double for Null Value"); + } + return delegate.getDouble(); + } + + @Override + public boolean isNull() + { + return isNullResult || delegate.isNull(); + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java new file mode 100644 index 000000000000..7d69ce9f9903 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableAggregatorFactory.java @@ -0,0 +1,72 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + + +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.Pair; +import io.druid.segment.BaseNullableColumnValueSelector; +import io.druid.segment.ColumnSelectorFactory; + +/** + * abstract class with functionality to wrap aggregator/bufferAggregator/combiner to make them Nullable. + * Implementations of {@link AggregatorFactory} which needs to Support Nullable Aggregations are encouraged + * to extend this class. + */ +public abstract class NullableAggregatorFactory extends AggregatorFactory +{ + @Override + public final Aggregator factorize(ColumnSelectorFactory metricFactory) + { + Pair pair = factorize2( + metricFactory); + return NullHandling.useDefaultValuesForNull() ? pair.lhs : new NullableAggregator(pair.lhs, pair.rhs); + } + + protected abstract Pair factorize2(ColumnSelectorFactory metricfactory); + + @Override + public final BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + Pair pair = factorizeBuffered2( + metricFactory); + return NullHandling.useDefaultValuesForNull() ? pair.lhs : new NullableBufferAggregator(pair.lhs, pair.rhs); + } + + protected abstract Pair factorizeBuffered2(ColumnSelectorFactory metricfactory); + + + @Override + public final AggregateCombiner makeAggregateCombiner() + { + AggregateCombiner combiner = makeAggregateCombiner2(); + return NullHandling.useDefaultValuesForNull() ? combiner : new NullableAggregateCombiner(combiner); + } + + protected abstract AggregateCombiner makeAggregateCombiner2(); + + @Override + public final int getMaxIntermediateSize() + { + return getMaxIntermediateSize2() + (NullHandling.useDefaultValuesForNull() ? 0 : Byte.BYTES); + } + + protected abstract int getMaxIntermediateSize2(); +} diff --git a/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java new file mode 100644 index 000000000000..cd27bc6039d0 --- /dev/null +++ b/processing/src/main/java/io/druid/query/aggregation/NullableBufferAggregator.java @@ -0,0 +1,114 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.aggregation; + +import io.druid.segment.BaseNullableColumnValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * The result of a NullableBufferAggregator will be null if all the values to be aggregated are null values or no values are aggregated at all. + * If any of the value is non-null, the result would be the aggregated value of the delegate aggregator. + * Note that the delegate aggregator is not required to perform check for isNull on the columnValueSelector as only non-null values + * will be passed to the delegate aggregator. + */ +public class NullableBufferAggregator implements BufferAggregator +{ + private static final byte IS_NULL_BYTE = (byte) 1; + private static final byte IS_NOT_NULL_BYTE = (byte) 0; + private final BufferAggregator delegate; + private final BaseNullableColumnValueSelector selector; + + + public NullableBufferAggregator(BufferAggregator delegate, BaseNullableColumnValueSelector selector) + { + this.delegate = delegate; + this.selector = selector; + } + + @Override + public void init(ByteBuffer buf, int position) + { + buf.put(position, IS_NULL_BYTE); + delegate.init(buf, position + Byte.BYTES); + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + boolean isNotNull = !selector.isNull(); + if (isNotNull) { + if (buf.get(position) == IS_NULL_BYTE) { + buf.put(position, IS_NOT_NULL_BYTE); + } + delegate.aggregate(buf, position + Byte.BYTES); + } + } + + @Override + @Nullable + public Object get(ByteBuffer buf, int position) + { + if (buf.get(position) == IS_NULL_BYTE) { + return null; + } + return delegate.get(buf, position + Byte.BYTES); + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return float for Null Value"); + } + return delegate.getFloat(buf, position + Byte.BYTES); + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return long for Null Value"); + } + return delegate.getLong(buf, position + Byte.BYTES); + } + + @Override + public double getDouble(ByteBuffer buf, int position) + { + if (isNull(buf, position)) { + throw new IllegalStateException("Cannot return double for Null Value"); + } + return delegate.getDouble(buf, position + Byte.BYTES); + } + + @Override + public boolean isNull(ByteBuffer buf, int position) + { + return buf.get(position) == IS_NULL_BYTE || delegate.isNull(buf, position + Byte.BYTES); + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java b/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java index bc19ba476e7a..d486ecad82bb 100644 --- a/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java +++ b/processing/src/main/java/io/druid/query/aggregation/ObjectAggregateCombiner.java @@ -26,4 +26,9 @@ */ public abstract class ObjectAggregateCombiner implements AggregateCombiner, ObjectColumnSelector { + @Override + public boolean isNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java index 171d4f39ef18..3d789d1d3dab 100644 --- a/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java @@ -22,6 +22,7 @@ import io.druid.guice.annotations.ExtensionPoint; import io.druid.java.util.common.Cacheable; +import javax.annotation.Nullable; import java.util.Comparator; import java.util.Map; import java.util.Set; @@ -36,6 +37,7 @@ public interface PostAggregator extends Cacheable Comparator getComparator(); + @Nullable Object compute(Map combinedAggregators); String getName(); diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index f3b173805f0a..6a809a441695 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -28,12 +28,13 @@ import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; -public abstract class SimpleDoubleAggregatorFactory extends AggregatorFactory +public abstract class SimpleDoubleAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -91,7 +92,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Double.BYTES; } @@ -152,7 +153,8 @@ public List requiredFields() } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java index a37e0226e0dd..cdc797658577 100644 --- a/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -26,12 +26,13 @@ import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Objects; -public abstract class SimpleFloatAggregatorFactory extends AggregatorFactory +public abstract class SimpleFloatAggregatorFactory extends NullableAggregatorFactory { protected final String name; protected final String fieldName; @@ -87,7 +88,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Float.BYTES; } @@ -99,7 +100,8 @@ public Comparator getComparator() } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { return object; } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java index 5fd15ae30c1b..7d37d5f2da1b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/DoubleCardinalityAggregatorColumnSelectorStrategy.java @@ -20,23 +20,33 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseDoubleColumnValueSelector; - +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class DoubleCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override - public void hashRow(BaseDoubleColumnValueSelector dimSelector, Hasher hasher) + public void hashRow(BaseDoubleColumnValueSelector selector, Hasher hasher) { - hasher.putDouble(dimSelector.getDouble()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + hasher.putDouble(selector.getDouble()); + } } @Override - public void hashValues(BaseDoubleColumnValueSelector dimSelector, HyperLogLogCollector collector) + public void hashValues(BaseDoubleColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(dimSelector.getDouble())).asBytes()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(Double.doubleToLongBits(selector.getDouble())).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java index b46261c7b15e..33b66019618b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/FloatCardinalityAggregatorColumnSelectorStrategy.java @@ -20,22 +20,33 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseFloatColumnValueSelector; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class FloatCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override public void hashRow(BaseFloatColumnValueSelector selector, Hasher hasher) { - hasher.putFloat(selector.getFloat()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + hasher.putFloat(selector.getFloat()); + } } @Override public void hashValues(BaseFloatColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashInt(Float.floatToIntBits(selector.getFloat())).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java index a666ed64e1a7..82879bba81ea 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/LongCardinalityAggregatorColumnSelectorStrategy.java @@ -20,22 +20,33 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.BaseLongColumnValueSelector; +/** + * if performance of this class appears to be a bottleneck for somebody, + * one simple way to improve it is to split it into two different classes, + * one that is used when {@link NullHandling.useDefaultValuesForNull()} is false, + * and one - when it's true, moving this computation out of the tight loop + */ public class LongCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy { @Override - public void hashRow(BaseLongColumnValueSelector dimSelector, Hasher hasher) + public void hashRow(BaseLongColumnValueSelector selector, Hasher hasher) { - hasher.putLong(dimSelector.getLong()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + hasher.putLong(selector.getLong()); + } } @Override - public void hashValues(BaseLongColumnValueSelector dimSelector, HyperLogLogCollector collector) + public void hashValues(BaseLongColumnValueSelector selector, HyperLogLogCollector collector) { - collector.add(CardinalityAggregator.hashFn.hashLong(dimSelector.getLong()).asBytes()); + if (NullHandling.useDefaultValuesForNull() || !selector.isNull()) { + collector.add(CardinalityAggregator.hashFn.hashLong(selector.getLong()).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java index 71661abf91b9..7e98ef7b96a3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/types/StringCardinalityAggregatorColumnSelectorStrategy.java @@ -20,6 +20,7 @@ package io.druid.query.aggregation.cardinality.types; import com.google.common.hash.Hasher; +import io.druid.common.config.NullHandling; import io.druid.hll.HyperLogLogCollector; import io.druid.query.aggregation.cardinality.CardinalityAggregator; import io.druid.segment.DimensionSelector; @@ -27,7 +28,8 @@ import java.util.Arrays; -public class StringCardinalityAggregatorColumnSelectorStrategy implements CardinalityAggregatorColumnSelectorStrategy +public class StringCardinalityAggregatorColumnSelectorStrategy + implements CardinalityAggregatorColumnSelectorStrategy { public static final String CARDINALITY_AGG_NULL_STRING = "\u0000"; public static final char CARDINALITY_AGG_SEPARATOR = '\u0001'; @@ -40,20 +42,34 @@ public void hashRow(DimensionSelector dimSelector, Hasher hasher) // nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases. if (size == 1) { final String value = dimSelector.lookupName(row.get(0)); - hasher.putUnencodedChars(nullToSpecial(value)); + if (NullHandling.useDefaultValuesForNull() || value != null) { + hasher.putUnencodedChars(nullToSpecial(value)); + } } else if (size != 0) { + boolean hasNonNullValue = false; final String[] values = new String[size]; for (int i = 0; i < size; ++i) { final String value = dimSelector.lookupName(row.get(i)); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. + if (!NullHandling.useDefaultValuesForNull() && !hasNonNullValue && value != null) { + hasNonNullValue = true; + } values[i] = nullToSpecial(value); } - // Values need to be sorted to ensure consistent multi-value ordering across different segments - Arrays.sort(values); - for (int i = 0; i < size; ++i) { - if (i != 0) { - hasher.putChar(CARDINALITY_AGG_SEPARATOR); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. + if (NullHandling.useDefaultValuesForNull() || hasNonNullValue) { + // Values need to be sorted to ensure consistent multi-value ordering across different segments + Arrays.sort(values); + for (int i = 0; i < size; ++i) { + if (i != 0) { + hasher.putChar(CARDINALITY_AGG_SEPARATOR); + } + hasher.putUnencodedChars(values[i]); } - hasher.putUnencodedChars(values[i]); } } } @@ -65,7 +81,12 @@ public void hashValues(DimensionSelector dimSelector, HyperLogLogCollector colle for (int i = 0; i < row.size(); i++) { int index = row.get(i); final String value = dimSelector.lookupName(index); - collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); + // SQL standard spec does not count null values, + // Skip counting null values when we are not replacing null with default value. + // A special value for null in case null handling is configured to use empty string for null. + if (NullHandling.useDefaultValuesForNull() || value != null) { + collector.add(CardinalityAggregator.hashFn.hashUnencodedChars(nullToSpecial(value)).asBytes()); + } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index be2dd5c84e4e..73f3f40a9964 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -26,17 +26,22 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleFirstAggregatorFactory extends AggregatorFactory +public class DoubleFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, @@ -75,20 +80,26 @@ public DoubleFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) + BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector ); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) + BaseDoubleColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector ); } @@ -99,13 +110,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("DoubleFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -116,46 +134,48 @@ public AggregatorFactory getCombiningFactory() return new DoubleFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Longs.BYTES, pair.rhs); - } - } + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Longs.BYTES, pair.rhs); + } + } - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -174,9 +194,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -219,7 +240,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Double.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 0b62e47aebb1..3360f0feac1d 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -26,17 +26,21 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +48,7 @@ import java.util.Map; import java.util.Objects; -public class FloatFirstAggregatorFactory extends AggregatorFactory +public class FloatFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Doubles.compare( ((SerializablePair) o1).rhs, @@ -73,21 +77,25 @@ public FloatFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + ColumnValueSelector columnValueSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + columnValueSelector + ), columnValueSelector); } @Override @@ -97,13 +105,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("FloatFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -114,46 +129,48 @@ public AggregatorFactory getCombiningFactory() return new FloatFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Longs.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Longs.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -172,9 +189,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? object : ((SerializablePair) object).rhs; } @Override @@ -214,7 +232,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Float.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index b000d9d5c363..7c09ddd37097 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -25,24 +25,29 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; -public class LongFirstAggregatorFactory extends AggregatorFactory +public class LongFirstAggregatorFactory extends NullableAggregatorFactory { public static final Comparator VALUE_COMPARATOR = (o1, o2) -> Longs.compare( ((SerializablePair) o1).rhs, @@ -66,21 +71,25 @@ public LongFirstAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongFirstAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongFirstAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongFirstBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongFirstBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override @@ -90,13 +99,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (lhs == null) { + return rhs; + } + if (rhs == null) { + return lhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) <= 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("LongFirstAggregatorFactory is not supported during ingestion for rollup"); } @@ -107,46 +123,48 @@ public AggregatorFactory getCombiningFactory() return new LongFirstAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - firstValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongFirstAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + firstValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongFirstBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long firstTime = buf.getLong(position); + if (pair.lhs < firstTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -165,9 +183,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? object : ((SerializablePair) object).rhs; } @Override @@ -207,7 +226,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES * 2; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 3241d49aee62..a0db56a6b4b3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -25,19 +25,24 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseDoubleColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -45,7 +50,7 @@ import java.util.Map; import java.util.Objects; -public class DoubleLastAggregatorFactory extends AggregatorFactory +public class DoubleLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; @@ -66,21 +71,25 @@ public DoubleLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new DoubleLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + doubleColumnSelector + ), doubleColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new DoubleLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseDoubleColumnValueSelector doubleColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new DoubleLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + doubleColumnSelector + ), doubleColumnSelector); } @Override @@ -90,13 +99,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("DoubleLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -107,46 +123,48 @@ public AggregatorFactory getCombiningFactory() return new DoubleLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new DoubleLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putDouble(position + Longs.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new DoubleLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putDouble(position + Longs.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -165,9 +183,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -211,7 +230,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Double.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 73d671b69c96..93e2efa716b8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -25,19 +25,24 @@ import com.google.common.primitives.Longs; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.FloatFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseFloatColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -45,7 +50,7 @@ import java.util.Map; import java.util.Objects; -public class FloatLastAggregatorFactory extends AggregatorFactory +public class FloatLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; @@ -64,21 +69,25 @@ public FloatLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new FloatLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + floatColumnSelector + ), floatColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new FloatLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseFloatColumnValueSelector floatColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new FloatLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + floatColumnSelector + ), floatColumnSelector); } @Override @@ -88,13 +97,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return FloatFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("FloatLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -105,46 +121,48 @@ public AggregatorFactory getCombiningFactory() return new FloatLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putFloat(position + Longs.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new FloatLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putFloat(position + Longs.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -163,9 +181,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -206,7 +225,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES + Float.BYTES; } diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 60f9669f5913..d2cf47bba0e4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -24,19 +24,24 @@ import com.google.common.base.Preconditions; import com.metamx.common.StringUtils; import io.druid.collections.SerializablePair; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.NullableAggregatorFactory; import io.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import io.druid.query.aggregation.first.LongFirstAggregatorFactory; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.BaseLongColumnValueSelector; +import io.druid.segment.BaseNullableColumnValueSelector; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; import io.druid.segment.column.Column; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Comparator; @@ -44,7 +49,7 @@ import java.util.Map; import java.util.Objects; -public class LongLastAggregatorFactory extends AggregatorFactory +public class LongLastAggregatorFactory extends NullableAggregatorFactory { private final String fieldName; private final String name; @@ -62,21 +67,25 @@ public LongLastAggregatorFactory( } @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - return new LongLastAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongLastAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - return new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), - metricFactory.makeColumnValueSelector(fieldName) - ); + BaseLongColumnValueSelector longColumnSelector = metricFactory.makeColumnValueSelector(fieldName); + return Pair.of( + new LongLastBufferAggregator( + metricFactory.makeColumnValueSelector(Column.TIME_COLUMN_NAME), + longColumnSelector + ), longColumnSelector); } @Override @@ -86,13 +95,20 @@ public Comparator getComparator() } @Override - public Object combine(Object lhs, Object rhs) + @Nullable + public Object combine(@Nullable Object lhs, @Nullable Object rhs) { + if (rhs == null) { + return lhs; + } + if (lhs == null) { + return rhs; + } return DoubleFirstAggregatorFactory.TIME_COMPARATOR.compare(lhs, rhs) > 0 ? lhs : rhs; } @Override - public AggregateCombiner makeAggregateCombiner() + public AggregateCombiner makeAggregateCombiner2() { throw new UOE("LongLastAggregatorFactory is not supported during ingestion for rollup"); } @@ -103,46 +119,48 @@ public AggregatorFactory getCombiningFactory() return new LongLastAggregatorFactory(name, name) { @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) + public Pair factorize2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = (SerializablePair) selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - lastValue = pair.rhs; - } - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongLastAggregator(null, null) + { + @Override + public void aggregate() + { + SerializablePair pair = (SerializablePair) selector.getObject(); + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + lastValue = pair.rhs; + } + } + }, selector); } @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + public Pair factorizeBuffered2(ColumnSelectorFactory metricFactory) { - final BaseObjectColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); - return new LongLastBufferAggregator(null, null) - { - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - buf.putLong(position, pair.lhs); - buf.putLong(position + Long.BYTES, pair.rhs); - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(name); + return Pair.of( + new LongLastBufferAggregator(null, null) + { + @Override + public void aggregate(ByteBuffer buf, int position) + { + SerializablePair pair = (SerializablePair) selector.getObject(); + long lastTime = buf.getLong(position); + if (pair.lhs >= lastTime) { + buf.putLong(position, pair.lhs); + buf.putLong(position + Long.BYTES, pair.rhs); + } + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }, selector); } }; } @@ -161,9 +179,10 @@ public Object deserialize(Object object) } @Override - public Object finalizeComputation(Object object) + @Nullable + public Object finalizeComputation(@Nullable Object object) { - return ((SerializablePair) object).rhs; + return object == null ? null : ((SerializablePair) object).rhs; } @Override @@ -203,7 +222,7 @@ public String getTypeName() } @Override - public int getMaxIntermediateSize() + public int getMaxIntermediateSize2() { return Long.BYTES * 2; } diff --git a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index ff2b6c838612..ecb70da4bf16 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; @@ -109,11 +110,21 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = 0.0; + Double retVal = NullHandling.defaultDoubleValue(); if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); + Number nextVal = (Number) fieldsIter.next().compute(values); + if (nextVal == null) { + // As per SQL standard if any of the value is null, arithmetic operators will return null. + return null; + } + retVal = nextVal.doubleValue(); while (fieldsIter.hasNext()) { - retVal = op.compute(retVal, ((Number) fieldsIter.next().compute(values)).doubleValue()); + nextVal = (Number) fieldsIter.next().compute(values); + if (nextVal == null) { + // As per SQL standard if any of the value is null, arithmetic operators will return null. + return null; + } + retVal = op.compute(retVal, (nextVal).doubleValue()); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 584d022675b5..95f23fc05e9f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -37,14 +39,7 @@ public class DoubleGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return ((Double) o).compareTo((Double) o1); - } - }; + private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); private final String name; private final List fields; @@ -81,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = Double.NEGATIVE_INFINITY; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); - while (fieldsIter.hasNext()) { - double other = ((Number) fieldsIter.next().compute(values)).doubleValue(); - if (other > retVal) { - retVal = other; - } + Double retVal = NullHandling.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) > 0) { + retVal = nextVal.doubleValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java index f66c81d9e769..9c47002e2ed8 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -22,7 +22,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -37,14 +39,7 @@ public class DoubleLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return ((Double) o).compareTo((Double) o1); - } - }; + private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); private final String name; private final List fields; @@ -81,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - double retVal = Double.POSITIVE_INFINITY; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).doubleValue(); - while (fieldsIter.hasNext()) { - double other = ((Number) fieldsIter.next().compute(values)).doubleValue(); - if (other < retVal) { - retVal = other; - } + Double retVal = NullHandling.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.doubleValue(), retVal) < 0) { + retVal = nextVal.doubleValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java index 695568c4dad4..567969fb2b93 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -23,7 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -38,14 +39,7 @@ public class LongGreatestPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((Number) o).longValue(), ((Number) o1).longValue()); - } - }; + private static final Comparator COMPARATOR = Comparators.naturalNullsFirst(); private final String name; private final List fields; @@ -82,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - long retVal = Long.MIN_VALUE; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).longValue(); - while (fieldsIter.hasNext()) { - long other = ((Number) fieldsIter.next().compute(values)).longValue(); - if (other > retVal) { - retVal = other; - } + Long retVal = NullHandling.useDefaultValuesForNull() ? Long.MIN_VALUE : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) > 0) { + retVal = nextVal.longValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java index f14bcd8228f3..1d6faaab9c28 100644 --- a/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -22,8 +22,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.query.Queries; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -38,14 +39,7 @@ public class LongLeastPostAggregator implements PostAggregator { - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(Object o, Object o1) - { - return Longs.compare(((Number) o).longValue(), ((Number) o1).longValue()); - } - }; + private static final Comparator COMPARATOR = Ordering.natural().nullsLast(); private final String name; private final List fields; @@ -82,14 +76,12 @@ public Comparator getComparator() public Object compute(Map values) { Iterator fieldsIter = fields.iterator(); - long retVal = Long.MAX_VALUE; - if (fieldsIter.hasNext()) { - retVal = ((Number) fieldsIter.next().compute(values)).longValue(); - while (fieldsIter.hasNext()) { - long other = ((Number) fieldsIter.next().compute(values)).longValue(); - if (other < retVal) { - retVal = other; - } + Long retVal = NullHandling.useDefaultValuesForNull() ? Long.MAX_VALUE : null; + while (fieldsIter.hasNext()) { + Number nextVal = ((Number) fieldsIter.next().compute(values)); + // Ignore NULL values and return the greatest out of non-null values. + if (nextVal != null && COMPARATOR.compare(nextVal.longValue(), retVal) < 0) { + retVal = nextVal.longValue(); } } return retVal; diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index c7ab3873b578..3fb3e852d676 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; @@ -106,7 +106,7 @@ private DimensionSelector filterWhiteList(DimensionSelector selector) } } else { for (int i = 0; i < selectorCardinality; i++) { - if (values.contains(Strings.nullToEmpty(selector.lookupName(i)))) { + if (values.contains(NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } @@ -137,7 +137,7 @@ public boolean apply(@Nullable String input) forwardMapping.defaultReturnValue(-1); final int[] reverseMapping = new int[maxPossibleFilteredCardinality]; for (int i = 0; i < selectorCardinality; i++) { - if (!values.contains(Strings.nullToEmpty(selector.lookupName(i)))) { + if (!values.contains(NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)))) { forwardMapping.put(i, count); reverseMapping[count++] = i; } diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index 4ebd407083f4..ce11703476be 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.filter.DimFilterUtils; import io.druid.segment.DimensionSelector; @@ -76,7 +76,7 @@ public DimensionSelector decorate(final DimensionSelector selector) @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(Strings.nullToEmpty(input)).matches(); + return compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches(); } } ); @@ -86,7 +86,8 @@ public boolean apply(@Nullable String input) final Int2IntOpenHashMap forwardMapping = new Int2IntOpenHashMap(); forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { - if (compiledRegex.matcher(Strings.nullToEmpty(selector.lookupName(i))).matches()) { + String val = NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)); + if (val != null && compiledRegex.matcher(val).matches()) { forwardMapping.put(i, count++); } } diff --git a/processing/src/main/java/io/druid/query/expression/ExprUtils.java b/processing/src/main/java/io/druid/query/expression/ExprUtils.java index 3a1d88702ae8..0a78c20e5d2e 100644 --- a/processing/src/main/java/io/druid/query/expression/ExprUtils.java +++ b/processing/src/main/java/io/druid/query/expression/ExprUtils.java @@ -22,6 +22,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.math.expr.Expr; +import org.apache.logging.log4j.util.Strings; import org.joda.time.Chronology; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -72,7 +73,10 @@ public static PeriodGranularity toPeriodGranularity( } else { Chronology chronology = timeZone == null ? ISOChronology.getInstanceUTC() : ISOChronology.getInstance(timeZone); final Object value = originArg.eval(bindings).value(); - origin = value != null ? new DateTime(value, chronology) : null; + origin = !(value == null || (value instanceof String && Strings.isBlank((String) value))) ? new DateTime( + value, + chronology + ) : null; } return new PeriodGranularity(period, origin, timeZone); diff --git a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java index c60400be33eb..d03ea8d13c7d 100644 --- a/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/LikeExprMacro.java @@ -19,7 +19,7 @@ package io.druid.query.expression; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -63,7 +63,7 @@ public Expr apply(final List args) } final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from( - Strings.nullToEmpty((String) patternExpr.getLiteralValue()), + NullHandling.nullToEmptyIfNeeded((String) patternExpr.getLiteralValue()), escapeChar ); diff --git a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java index f9a4273a05ae..940630c250c6 100644 --- a/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java @@ -19,7 +19,7 @@ package io.druid.query.expression; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -63,9 +63,10 @@ class RegexpExtractExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - final Matcher matcher = pattern.matcher(Strings.nullToEmpty(arg.eval(bindings).asString())); + String s = arg.eval(bindings).asString(); + final Matcher matcher = pattern.matcher(NullHandling.nullToEmptyIfNeeded(s)); final String retVal = matcher.find() ? matcher.group(index) : null; - return ExprEval.of(Strings.emptyToNull(retVal)); + return ExprEval.of(NullHandling.emptyToNullIfNeeded(retVal)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java index 55fee2e237bb..875e5e7363c5 100644 --- a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java @@ -62,7 +62,6 @@ public String apply(@Nullable Object value) if (value == null) { return null; } - if (value instanceof Number) { return bucket(((Number) value).doubleValue()); } else if (value instanceof String) { @@ -78,7 +77,6 @@ public String apply(@Nullable String value) if (value == null) { return null; } - try { return bucket(Double.parseDouble(value)); } diff --git a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java index bd762574efd1..1bacaab5ec52 100644 --- a/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java +++ b/processing/src/main/java/io/druid/query/extraction/FunctionalExtraction.java @@ -21,7 +21,7 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -52,9 +52,9 @@ public FunctionalExtraction( ) { this.retainMissingValue = retainMissingValue; - this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandling.emptyToNullIfNeeded(replaceMissingValueWith); Preconditions.checkArgument( - !(this.retainMissingValue && !Strings.isNullOrEmpty(this.replaceMissingValueWith)), + !(this.retainMissingValue && !(this.replaceMissingValueWith == null)), "Cannot specify a [replaceMissingValueWith] and set [retainMissingValue] to true" ); @@ -69,7 +69,7 @@ public FunctionalExtraction( public String apply(@Nullable String dimValue) { final String retval = extractionFunction.apply(dimValue); - return Strings.isNullOrEmpty(retval) ? Strings.emptyToNull(dimValue) : retval; + return NullHandling.isNullOrEquivalent(retval) ? NullHandling.emptyToNullIfNeeded(dimValue) : retval; } }; } else { @@ -79,8 +79,10 @@ public String apply(@Nullable String dimValue) @Override public String apply(@Nullable String dimValue) { - final String retval = extractionFunction.apply(dimValue); - return Strings.isNullOrEmpty(retval) ? FunctionalExtraction.this.replaceMissingValueWith : retval; + final String retval = NullHandling.emptyToNullIfNeeded(extractionFunction.apply(dimValue)); + return retval == null + ? FunctionalExtraction.this.replaceMissingValueWith + : retval; } }; } diff --git a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java index 1bf700feaaf0..0967bdfac0ad 100644 --- a/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/IdentityExtractionFn.java @@ -19,7 +19,7 @@ package io.druid.query.extraction; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -42,14 +42,14 @@ public byte[] getCacheKey() @Nullable public String apply(@Nullable Object value) { - return value == null ? null : Strings.emptyToNull(value.toString()); + return value == null ? null : NullHandling.emptyToNullIfNeeded(value.toString()); } @Override @Nullable public String apply(@Nullable String value) { - return Strings.emptyToNull(value); + return NullHandling.emptyToNullIfNeeded(value); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java index 1d4e479acd8d..5aec937fba34 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavaScriptExtractionFn.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.js.JavaScriptConfig; import org.mozilla.javascript.Context; @@ -113,7 +113,7 @@ public byte[] getCacheKey() public String apply(@Nullable Object value) { checkAndCompileScript(); - return Strings.emptyToNull(fn.apply(value)); + return NullHandling.emptyToNullIfNeeded(fn.apply(value)); } /** @@ -126,7 +126,6 @@ private void checkAndCompileScript() // JavaScript configuration should be checked when it's actually used because someone might still want Druid // nodes to be able to deserialize JavaScript-based objects even though JavaScript is disabled. Preconditions.checkState(config.isEnabled(), "JavaScript is disabled"); - synchronized (config) { if (fn == null) { fn = compile(function); @@ -139,7 +138,7 @@ private void checkAndCompileScript() @Nullable public String apply(@Nullable String value) { - return this.apply((Object) Strings.emptyToNull(value)); + return this.apply((Object) NullHandling.emptyToNullIfNeeded(value)); } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java index da2a8e3bc702..b0dddf121b73 100644 --- a/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/LowerExtractionFn.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -52,7 +53,7 @@ public LowerExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (Strings.isNullOrEmpty(key)) { + if (NullHandling.isNullOrEquivalent(key)) { return null; } return key.toLowerCase(locale); @@ -73,7 +74,9 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { + //CHECKSTYLE.OFF: Regexp byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_LOWER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java index b818bd8cd50c..ccdae8f0932f 100644 --- a/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java +++ b/processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java @@ -76,7 +76,9 @@ public List unapply(final String value) { @Override public boolean apply(@Nullable String key) { + //CHECKSTYLE.OFF: Regexp return map.get(key).equals(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp } }).keySet()); diff --git a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java index 4d121be031fc..5c1054afad91 100644 --- a/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/MatchingDimExtractionFn.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -62,7 +62,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { // We'd return null whether or not the pattern matched return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index 3a1b0a5cc293..feab409d4ba1 100644 --- a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -106,13 +106,14 @@ public byte[] getCacheKey() public String apply(@Nullable String dimValue) { final String retVal; - final Matcher matcher = pattern.matcher(Strings.nullToEmpty(dimValue)); - if (matcher.find()) { + String val = NullHandling.nullToEmptyIfNeeded(dimValue); + final Matcher matcher = val == null ? null : pattern.matcher(val); + if (matcher != null && matcher.find()) { retVal = matcher.group(index); } else { retVal = replaceMissingValue ? replaceMissingValueWith : dimValue; } - return Strings.emptyToNull(retVal); + return NullHandling.emptyToNullIfNeeded(retVal); } @JsonProperty("expr") diff --git a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java index 1f005523925b..3e30daa6f5f6 100644 --- a/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.query.search.SearchQuerySpec; import javax.annotation.Nullable; @@ -64,7 +64,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - return searchQuerySpec.accept(dimValue) ? Strings.emptyToNull(dimValue) : null; + return searchQuerySpec.accept(dimValue) ? NullHandling.emptyToNullIfNeeded(dimValue) : null; } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java index 168cad01ea24..ba21754774fe 100644 --- a/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StringFormatExtractionFn.java @@ -107,7 +107,9 @@ public String apply(@Nullable String value) value = ""; } } + //CHECKSTYLE.OFF: Regexp return Strings.emptyToNull(StringUtils.format(format, value)); + //CHECKSTYLE.ON: Regexp } @Override diff --git a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java index 5ad88fa38fce..fa08a9741f69 100644 --- a/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/StrlenExtractionFn.java @@ -20,6 +20,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; +import io.druid.common.config.NullHandling; import javax.annotation.Nullable; @@ -40,6 +41,9 @@ public static StrlenExtractionFn instance() @Override public String apply(@Nullable String value) { + if (!NullHandling.useDefaultValuesForNull() && value == null) { + return null; + } return String.valueOf(value == null ? 0 : value.length()); } diff --git a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java index 75b251f18faf..c980aaa920fb 100644 --- a/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SubstringDimExtractionFn.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -63,7 +63,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 0aa7fdeb8acf..c69a21558a68 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.ibm.icu.text.SimpleDateFormat; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -77,7 +77,7 @@ public byte[] getCacheKey() @Override public String apply(@Nullable String dimValue) { - if (Strings.isNullOrEmpty(dimValue)) { + if (NullHandling.isNullOrEquivalent(dimValue)) { return null; } diff --git a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java index 34368f7f0fd6..ec2b914d3806 100644 --- a/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/UpperExtractionFn.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; @@ -51,7 +52,7 @@ public UpperExtractionFn(@JsonProperty("locale") String localeString) @Override public String apply(@Nullable String key) { - if (Strings.isNullOrEmpty(key)) { + if (NullHandling.isNullOrEquivalent(key)) { return null; } return key.toUpperCase(locale); @@ -72,7 +73,9 @@ public ExtractionType getExtractionType() @Override public byte[] getCacheKey() { + //CHECKSTYLE.OFF: Regexp byte[] localeBytes = StringUtils.toUtf8(Strings.nullToEmpty(localeString)); + //CHECKSTYLE.ON: Regexp return ByteBuffer.allocate(2 + localeBytes.length) .put(ExtractionCacheHelper.CACHE_TYPE_ID_UPPER) .put((byte) 0XFF) diff --git a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java index f41e6c6a0242..a99d8a52de4f 100644 --- a/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/DoubleValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseDoubleColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.filter.BooleanValueMatcher; public class DoubleValueMatcherColumnSelectorStrategy @@ -33,7 +32,20 @@ public ValueMatcher makeValueMatcher(final BaseDoubleColumnValueSelector selecto { final Double matchVal = DimensionHandlerUtils.convertObjectToDouble(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final long matchValLongBits = Double.doubleToLongBits(matchVal); @@ -65,6 +77,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyDouble(selector.getDouble()); } @@ -80,13 +95,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final BaseDoubleColumnValueSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Double.toString(selector.getDouble())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Double.toString(selector.getDouble())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java index c28b8d376873..f1e2874ea6ee 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidDoublePredicate.java @@ -26,5 +26,25 @@ public interface DruidDoublePredicate DruidDoublePredicate ALWAYS_TRUE = input -> true; + DruidDoublePredicate MATCH_NULL_ONLY = new DruidDoublePredicate() + { + @Override + public boolean applyDouble(double input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyDouble(double input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java index 2aa2674dd246..63de55c2f9f0 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidFloatPredicate.java @@ -26,5 +26,25 @@ public interface DruidFloatPredicate { DruidFloatPredicate ALWAYS_FALSE = input -> false; + DruidFloatPredicate MATCH_NULL_ONLY = new DruidFloatPredicate() + { + @Override + public boolean applyFloat(float input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyFloat(float input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java index 2afca5c40e10..4175c6bcbe18 100644 --- a/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java +++ b/processing/src/main/java/io/druid/query/filter/DruidLongPredicate.java @@ -28,5 +28,25 @@ public interface DruidLongPredicate DruidLongPredicate ALWAYS_TRUE = input -> true; + DruidLongPredicate MATCH_NULL_ONLY = new DruidLongPredicate() + { + @Override + public boolean applyLong(long input) + { + return false; + } + + @Override + public boolean applyNull() + { + return true; + } + }; + boolean applyLong(long input); + + default boolean applyNull() + { + return false; + } } diff --git a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java index af7a2b215ed9..5628e09bf92d 100644 --- a/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/FloatValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseFloatColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.filter.BooleanValueMatcher; public class FloatValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy @@ -32,7 +31,20 @@ public ValueMatcher makeValueMatcher(final BaseFloatColumnValueSelector selector { final Float matchVal = DimensionHandlerUtils.convertObjectToFloat(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final int matchValIntBits = Float.floatToIntBits(matchVal); @@ -64,6 +76,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyFloat(selector.getFloat()); } @@ -79,13 +94,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final BaseFloatColumnValueSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Float.toString(selector.getFloat())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Float.toString(selector.getFloat())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/InDimFilter.java b/processing/src/main/java/io/druid/query/filter/InDimFilter.java index d12d0a552742..4e56d7eda92c 100644 --- a/processing/src/main/java/io/druid/query/filter/InDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/InDimFilter.java @@ -21,19 +21,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.extraction.ExtractionFn; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.lookup.LookupExtractor; @@ -51,6 +51,8 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; public class InDimFilter implements DimFilter { @@ -58,7 +60,7 @@ public class InDimFilter implements DimFilter // Hashing threshold is not applied to String for now, String still uses ImmutableSortedSet public static final int NUMERIC_HASHING_THRESHOLD = 16; - private final ImmutableSortedSet values; + private final SortedSet values; private final String dimension; private final ExtractionFn extractionFn; private final Supplier longPredicateSupplier; @@ -74,19 +76,11 @@ public InDimFilter( { Preconditions.checkNotNull(dimension, "dimension can not be null"); Preconditions.checkArgument(values != null && !values.isEmpty(), "values can not be null or empty"); - this.values = ImmutableSortedSet.copyOf( - Iterables.transform( - values, new Function() - { - @Override - public String apply(String input) - { - return Strings.nullToEmpty(input); - } - } - ) - ); + this.values = new TreeSet<>(Comparators.naturalNullsFirst()); + for (String value : values) { + this.values.add(NullHandling.emptyToNullIfNeeded(value)); + } this.dimension = dimension; this.extractionFn = extractionFn; this.longPredicateSupplier = getLongPredicateSupplier(); @@ -119,14 +113,20 @@ public byte[] getCacheKey() final byte[][] valuesBytes = new byte[values.size()][]; int valuesBytesSize = 0; int index = 0; + boolean hasNull = false; for (String value : values) { + if (value == null) { + hasNull = true; + } + //CHECKSTYLE.OFF: Regexp valuesBytes[index] = StringUtils.toUtf8(Strings.nullToEmpty(value)); + //CHECKSTYLE.ON: Regexp valuesBytesSize += valuesBytes[index].length + 1; ++index; } byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - ByteBuffer filterCacheKey = ByteBuffer.allocate(3 + ByteBuffer filterCacheKey = ByteBuffer.allocate(5 + dimensionBytes.length + valuesBytesSize + extractionFnBytes.length) @@ -134,6 +134,8 @@ public byte[] getCacheKey() .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(extractionFnBytes) + .put(DimFilterUtils.STRING_SEPARATOR) + .put(hasNull ? (byte) 1 : (byte) 0) .put(DimFilterUtils.STRING_SEPARATOR); for (byte[] bytes : valuesBytes) { filterCacheKey.put(bytes) @@ -165,7 +167,9 @@ private InDimFilter optimizeLookup() // We cannot do an unapply()-based optimization if the selector value // and the replaceMissingValuesWith value are the same, since we have to match on // all values that are not present in the lookup. + //CHECKSTYLE.OFF: Regexp final String convertedValue = Strings.emptyToNull(value); + //CHECKSTYLE.ON: Regexp if (!exFn.isRetainMissingValue() && Objects.equals(convertedValue, exFn.getReplaceMissingValueWith())) { return this; } @@ -175,7 +179,7 @@ private InDimFilter optimizeLookup() // there may be row values that match the selector value but are not included // in the lookup map. Match on the selector value as well. // If the selector value is overwritten in the lookup map, don't add selector value to keys. - if (exFn.isRetainMissingValue() && lookup.apply(convertedValue) == null) { + if (exFn.isRetainMissingValue() && NullHandling.isNullOrEquivalent(lookup.apply(convertedValue))) { keys.add(convertedValue); } } @@ -210,7 +214,9 @@ public RangeSet getDimensionRangeSet(String dimension) } RangeSet retSet = TreeRangeSet.create(); for (String value : values) { + //CHECKSTYLE.OFF: Regexp retSet.add(Range.singleton(Strings.nullToEmpty(value))); + //CHECKSTYLE.ON: Regexp } return retSet; } @@ -260,9 +266,11 @@ public String toString() if (extractionFn != null) { builder.append(")"); } - - builder.append(" IN (").append(Joiner.on(", ").join(values)).append(")"); - + //CHECKSTYLE.OFF: Regexp + builder.append(" IN (") + .append(Joiner.on(", ").join(Iterables.transform(values, input -> Strings.nullToEmpty(input)))) + .append(")"); + //CHECKSTYLE.ON: Regexp return builder.toString(); } diff --git a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java index 5a98bc5d2327..8a55ffbd4a68 100644 --- a/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/LikeDimFilter.java @@ -23,10 +23,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.collect.RangeSet; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.data.Indexed; @@ -98,7 +98,7 @@ private LikeMatcher( ) { this.suffixMatch = Preconditions.checkNotNull(suffixMatch, "suffixMatch"); - this.prefix = Strings.nullToEmpty(prefix); + this.prefix = NullHandling.nullToEmptyIfNeeded(prefix); this.pattern = Preconditions.checkNotNull(pattern, "pattern"); } @@ -151,7 +151,8 @@ private static void addPatternCharacter(final StringBuilder patternBuilder, fina public boolean matches(@Nullable final String s) { - return pattern.matcher(Strings.nullToEmpty(s)).matches(); + String val = NullHandling.nullToEmptyIfNeeded(s); + return val != null && pattern.matcher(val).matches(); } /** @@ -165,7 +166,7 @@ public boolean matchesSuffixOnly(final Indexed strings, final int i) return true; } else if (suffixMatch == SuffixMatch.MATCH_EMPTY) { final String s = strings.get(i); - return (s == null ? 0 : s.length()) == prefix.length(); + return s == null ? matches(null) : s.length() == prefix.length(); } else { // suffixMatch is MATCH_PATTERN final String s = strings.get(i); @@ -181,23 +182,9 @@ public DruidPredicateFactory predicateFactory(final ExtractionFn extractionFn) public Predicate makeStringPredicate() { if (extractionFn != null) { - return new Predicate() - { - @Override - public boolean apply(String input) - { - return matches(extractionFn.apply(input)); - } - }; + return input -> matches(extractionFn.apply(input)); } else { - return new Predicate() - { - @Override - public boolean apply(String input) - { - return matches(input); - } - }; + return input -> matches(input); } } diff --git a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java index 15a62bd50631..9b7966d7dd8f 100644 --- a/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/LongValueMatcherColumnSelectorStrategy.java @@ -22,7 +22,6 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.BaseLongColumnValueSelector; import io.druid.segment.DimensionHandlerUtils; -import io.druid.segment.filter.BooleanValueMatcher; public class LongValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy @@ -32,7 +31,20 @@ public ValueMatcher makeValueMatcher(final BaseLongColumnValueSelector selector, { final Long matchVal = DimensionHandlerUtils.convertObjectToLong(value); if (matchVal == null) { - return BooleanValueMatcher.of(false); + return new ValueMatcher() + { + @Override + public boolean matches() + { + return selector.isNull(); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("selector", selector); + } + }; } final long matchValLong = matchVal; return new ValueMatcher() @@ -63,6 +75,9 @@ public ValueMatcher makeValueMatcher( @Override public boolean matches() { + if (selector.isNull()) { + return predicate.applyNull(); + } return predicate.applyLong(selector.getLong()); } @@ -78,13 +93,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public ValueGetter makeValueGetter(final BaseLongColumnValueSelector selector) { - return new ValueGetter() - { - @Override - public String[] get() - { - return new String[]{Long.toString(selector.getLong())}; + return () -> { + if (selector.isNull()) { + return null; } + return new String[]{Long.toString(selector.getLong())}; }; } } diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index d13971a8ea28..8c18a99ccbba 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -25,12 +25,12 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import com.google.common.collect.TreeRangeSet; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionFn; @@ -38,6 +38,7 @@ import io.druid.segment.filter.SelectorFilter; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Objects; /** @@ -64,7 +65,7 @@ public SelectorDimFilter( Preconditions.checkArgument(dimension != null, "dimension must not be null"); this.dimension = dimension; - this.value = Strings.nullToEmpty(value); + this.value = NullHandling.emptyToNullIfNeeded(value); this.extractionFn = extractionFn; } @@ -75,8 +76,10 @@ public byte[] getCacheKey() byte[] valueBytes = (value == null) ? new byte[]{} : StringUtils.toUtf8(value); byte[] extractionFnBytes = extractionFn == null ? new byte[0] : extractionFn.getCacheKey(); - return ByteBuffer.allocate(3 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) + return ByteBuffer.allocate(6 + dimensionBytes.length + valueBytes.length + extractionFnBytes.length) .put(DimFilterUtils.SELECTOR_CACHE_ID) + .put(value == null ? (byte) 1 : (byte) 0) + .put(DimFilterUtils.STRING_SEPARATOR) .put(dimensionBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(valueBytes) @@ -88,7 +91,7 @@ public byte[] getCacheKey() @Override public DimFilter optimize() { - return new InDimFilter(dimension, ImmutableList.of(value), extractionFn).optimize(); + return new InDimFilter(dimension, Arrays.asList(value), extractionFn).optimize(); } @Override @@ -97,14 +100,13 @@ public Filter toFilter() if (extractionFn == null) { return new SelectorFilter(dimension, value); } else { - final String valueOrNull = Strings.emptyToNull(value); final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override public Predicate makeStringPredicate() { - return Predicates.equalTo(valueOrNull); + return Predicates.equalTo(value); } @Override @@ -188,7 +190,9 @@ public RangeSet getDimensionRangeSet(String dimension) return null; } RangeSet retSet = TreeRangeSet.create(); + //CHECKSTYLE.OFF: Regexp retSet.add(Range.singleton(Strings.nullToEmpty(value))); + //CHECKSTYLE.ON: Regexp return retSet; } @@ -211,6 +215,10 @@ private void initLongPredicate() if (longPredicate != null) { return; } + if (value == null) { + longPredicate = DruidLongPredicate.MATCH_NULL_ONLY; + return; + } final Long valueAsLong = GuavaUtils.tryParseLong(value); if (valueAsLong == null) { longPredicate = DruidLongPredicate.ALWAYS_FALSE; @@ -231,6 +239,11 @@ private void initFloatPredicate() if (floatPredicate != null) { return; } + + if (value == null) { + floatPredicate = DruidFloatPredicate.MATCH_NULL_ONLY; + return; + } final Float valueAsFloat = Floats.tryParse(value); if (valueAsFloat == null) { @@ -251,6 +264,10 @@ private void initDoublePredicate() if (druidDoublePredicate != null) { return; } + if (value == null) { + druidDoublePredicate = DruidDoublePredicate.MATCH_NULL_ONLY; + return; + } final Double aDouble = Doubles.tryParse(value); if (aDouble == null) { diff --git a/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java index e380b1162ee3..aa3dfe14aa4f 100644 --- a/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/filter/StringValueMatcherColumnSelectorStrategy.java @@ -20,7 +20,6 @@ package io.druid.query.filter; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; @@ -28,19 +27,11 @@ public class StringValueMatcherColumnSelectorStrategy implements ValueMatcherColumnSelectorStrategy { private static final String[] NULL_VALUE = new String[]{null}; - private static final ValueGetter NULL_VALUE_GETTER = new ValueGetter() - { - @Override - public String[] get() - { - return NULL_VALUE; - } - }; + private static final ValueGetter NULL_VALUE_GETTER = () -> NULL_VALUE; @Override public ValueMatcher makeValueMatcher(final DimensionSelector selector, String value) { - value = Strings.emptyToNull(value); if (selector.getValueCardinality() == 0) { return BooleanValueMatcher.of(value == null); } else { @@ -68,22 +59,17 @@ public ValueGetter makeValueGetter(final DimensionSelector selector) if (selector.getValueCardinality() == 0) { return NULL_VALUE_GETTER; } else { - return new ValueGetter() - { - @Override - public String[] get() - { - final IndexedInts row = selector.getRow(); - final int size = row.size(); - if (size == 0) { - return NULL_VALUE; - } else { - String[] values = new String[size]; - for (int i = 0; i < size; ++i) { - values[i] = Strings.emptyToNull(selector.lookupName(row.get(i))); - } - return values; + return () -> { + final IndexedInts row = selector.getRow(); + final int size = row.size(); + if (size == 0) { + return NULL_VALUE; + } else { + String[] values = new String[size]; + for (int i = 0; i < size; ++i) { + values[i] = selector.lookupName(row.get(i)); } + return values; } }; } diff --git a/processing/src/main/java/io/druid/query/filter/ValueGetter.java b/processing/src/main/java/io/druid/query/filter/ValueGetter.java index 72000a4756da..c478ce971b37 100644 --- a/processing/src/main/java/io/druid/query/filter/ValueGetter.java +++ b/processing/src/main/java/io/druid/query/filter/ValueGetter.java @@ -19,6 +19,8 @@ package io.druid.query.filter; +import javax.annotation.Nullable; + /** */ public interface ValueGetter @@ -27,5 +29,6 @@ public interface ValueGetter // converted to strings. We should also add functions // for these and modify ColumnComparisonFilter to handle // comparing Long and Float columns to eachother. + @Nullable String[] get(); } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 9ca527ea17a1..50335907ade9 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -58,6 +58,7 @@ import io.druid.query.ordering.StringComparators; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; @@ -527,19 +528,19 @@ private static int compareDims(List dimensions, Row lhs, Row rhs) for (DimensionSpec dimension : dimensions) { final int dimCompare; if (dimension.getOutputType() == ValueType.LONG) { - dimCompare = Long.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).longValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).longValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToLong(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToLong(rhs.getRaw(dimension.getOutputName())) ); } else if (dimension.getOutputType() == ValueType.FLOAT) { - dimCompare = Float.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).floatValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).floatValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToFloat(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToFloat(rhs.getRaw(dimension.getOutputName())) ); } else if (dimension.getOutputType() == ValueType.DOUBLE) { - dimCompare = Double.compare( - ((Number) lhs.getRaw(dimension.getOutputName())).doubleValue(), - ((Number) rhs.getRaw(dimension.getOutputName())).doubleValue() + dimCompare = Comparators.naturalNullsFirst().compare( + DimensionHandlerUtils.convertObjectToDouble(lhs.getRaw(dimension.getOutputName())), + DimensionHandlerUtils.convertObjectToDouble(rhs.getRaw(dimension.getOutputName())) ); } else { dimCompare = ((Ordering) Comparators.naturalNullsFirst()).compare( diff --git a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java index 70e53fa37d32..6bd9acb99d70 100644 --- a/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/RowBasedColumnSelectorFactory.java @@ -20,9 +20,9 @@ package io.druid.query.groupby; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.Row; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; @@ -30,6 +30,7 @@ import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnValueSelector; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.DimensionSelector; import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; @@ -222,13 +223,14 @@ public ValueMatcher makeValueMatcher(final String value) @Override public boolean matches() { - final List dimensionValues = row.get().getDimension(dimension); + Row row = RowBasedColumnSelectorFactory.this.row.get(); + final List dimensionValues = row.getDimension(dimension); if (dimensionValues == null || dimensionValues.isEmpty()) { return value == null; } for (String dimensionValue : dimensionValues) { - if (Objects.equals(Strings.emptyToNull(dimensionValue), value)) { + if (Objects.equals(NullHandling.emptyToNullIfNeeded(dimensionValue), value)) { return true; } } @@ -253,7 +255,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (Objects.equals(extractionFn.apply(Strings.emptyToNull(dimensionValue)), value)) { + if (Objects.equals(extractionFn.apply(NullHandling.emptyToNullIfNeeded(dimensionValue)), value)) { return true; } } @@ -286,7 +288,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(Strings.emptyToNull(dimensionValue))) { + if (predicate.apply(NullHandling.emptyToNullIfNeeded(dimensionValue))) { return true; } } @@ -312,7 +314,7 @@ public boolean matches() } for (String dimensionValue : dimensionValues) { - if (predicate.apply(extractionFn.apply(Strings.emptyToNull(dimensionValue)))) { + if (predicate.apply(extractionFn.apply(NullHandling.emptyToNullIfNeeded(dimensionValue)))) { return true; } } @@ -338,7 +340,7 @@ public int getValueCardinality() @Override public String lookupName(int id) { - final String value = Strings.emptyToNull(row.get().getDimension(dimension).get(id)); + final String value = NullHandling.emptyToNullIfNeeded(row.get().getDimension(dimension).get(id)); return extractionFn == null ? value : extractionFn.apply(value); } @@ -375,6 +377,7 @@ public Class classOfObject() return Object.class; } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -397,6 +400,13 @@ public long getLong() return row.get().getTimestampFromEpoch(); } + @Override + public boolean isNull() + { + // Time column never has null values + return false; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { @@ -410,19 +420,25 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public double getDouble() { - return row.get().getMetric(columnName).doubleValue(); + return DimensionHandlerUtils.nullToZeroDouble(row.get().getMetric(columnName)).doubleValue(); + } + + @Override + public boolean isNull() + { + return row.get().getRaw(columnName) == null; } @Override public float getFloat() { - return row.get().getMetric(columnName).floatValue(); + return DimensionHandlerUtils.nullToZeroDouble(row.get().getMetric(columnName)).floatValue(); } @Override public long getLong() { - return row.get().getMetric(columnName).longValue(); + return DimensionHandlerUtils.nullToZeroDouble(row.get().getMetric(columnName)).longValue(); } @Nullable diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java index 7a39b5e34cf1..bd6cf4a0d29e 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java @@ -21,9 +21,9 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; -import io.druid.java.util.common.parsers.CloseableIterator; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.common.parsers.CloseableIterator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.BufferAggregator; import io.druid.query.groupby.epinephelinae.column.GroupByColumnSelectorStrategy; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferHashGrouper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferHashGrouper.java index 1a8778a29e21..f5a28a7a5729 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferHashGrouper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferHashGrouper.java @@ -22,9 +22,9 @@ import com.google.common.base.Supplier; import com.google.common.collect.Iterators; import com.google.common.primitives.Ints; -import io.druid.java.util.common.parsers.CloseableIterator; import io.druid.java.util.common.CloseableIterators; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.parsers.CloseableIterator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.ColumnSelectorFactory; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferHashTable.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferHashTable.java index bcececdfd5cc..66567983ab10 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferHashTable.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferHashTable.java @@ -296,7 +296,7 @@ protected int findBucket( final int startBucket = keyHash % buckets; int bucket = startBucket; - outer: + outer: while (true) { final int bucketOffset = bucket * bucketSizeWithHash; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 6db22e1efb19..bc4abfa8d480 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -20,12 +20,12 @@ package io.druid.query.groupby.epinephelinae; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import io.druid.collections.NonBlockingPool; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; @@ -124,8 +124,8 @@ public static Sequence process( final ResourceHolder bufferHolder = intermediateResultsBufferPool.take(); - final String fudgeTimestampString = Strings.emptyToNull( - query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, "") + final String fudgeTimestampString = NullHandling.emptyToNullIfNeeded( + query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, null) ); final DateTime fudgeTimestamp = fudgeTimestampString == null @@ -685,19 +685,16 @@ private static void convertRowTypesToOutputTypes(List dimensionSp (dimName, baseVal) -> { switch (outputType) { case STRING: - baseVal = baseVal == null ? "" : baseVal.toString(); + baseVal = DimensionHandlerUtils.convertObjectToString(baseVal); break; case LONG: baseVal = DimensionHandlerUtils.convertObjectToLong(baseVal); - baseVal = baseVal == null ? 0L : baseVal; break; case FLOAT: baseVal = DimensionHandlerUtils.convertObjectToFloat(baseVal); - baseVal = baseVal == null ? 0.f : baseVal; break; case DOUBLE: baseVal = DimensionHandlerUtils.convertObjectToDouble(baseVal); - baseVal = baseVal == null ? 0.d : baseVal; break; default: throw new IAE("Unsupported type: " + outputType); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouper.java index 12c50ce940c0..d9d748da66e8 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouper.java @@ -22,10 +22,10 @@ import com.google.common.base.Supplier; import com.google.common.collect.Iterators; import com.google.common.primitives.Ints; -import io.druid.java.util.common.parsers.CloseableIterator; import io.druid.java.util.common.CloseableIterators; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.parsers.CloseableIterator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.ColumnSelectorFactory; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java index d043a9070d76..65f4e13ff7ae 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java @@ -289,11 +289,11 @@ private int computeRequiredBufferNum(int numChildNodes, int combineDegree) * Recursively build a combining tree in a bottom-up manner. Each node of the tree is a task that combines input * iterators asynchronously. * - * @param childIterators all iterators of the child level - * @param bufferSupplier combining buffer supplier - * @param combiningFactories array of combining aggregator factories - * @param combineDegree combining degree for the current level - * @param dictionary merged dictionary + * @param childIterators all iterators of the child level + * @param bufferSupplier combining buffer supplier + * @param combiningFactories array of combining aggregator factories + * @param combineDegree combining degree for the current level + * @param dictionary merged dictionary * * @return a pair of a list of iterators of the current level in the combining tree and a list of futures of all * executed combining tasks diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 290cf13753bf..1b25a37c43a0 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -34,6 +33,7 @@ import com.google.common.primitives.Longs; import com.google.common.util.concurrent.ListeningExecutorService; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.common.utils.IntArrayUtils; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; @@ -42,6 +42,7 @@ import io.druid.java.util.common.Pair; import io.druid.java.util.common.granularity.AllGranularity; import io.druid.java.util.common.guava.Accumulator; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.BaseQuery; import io.druid.query.ColumnSelectorPlus; import io.druid.query.aggregation.AggregatorFactory; @@ -441,7 +442,7 @@ public Row apply(Grouper.Entry entry) Object dimVal = entry.getKey().getKey()[i]; theMap.put( query.getDimensions().get(i - dimStart).getOutputName(), - dimVal instanceof String ? Strings.emptyToNull((String) dimVal) : dimVal + dimVal instanceof String ? NullHandling.emptyToNullIfNeeded((String) dimVal) : dimVal ); } @@ -530,18 +531,11 @@ private static class StringInputRawSupplierColumnSelectorStrategy @Override public Supplier makeInputRawSupplier(DimensionSelector selector) { - return new Supplier() - { - @Override - public Comparable get() - { - final String value; - IndexedInts index = selector.getRow(); - value = index.size() == 0 - ? "" - : selector.lookupName(index.get(0)); - return Strings.nullToEmpty(value); - } + return () -> { + IndexedInts index = selector.getRow(); + return index.size() == 0 + ? null + : selector.lookupName(index.get(0)); }; } } @@ -608,28 +602,19 @@ private static Function[] makeValueConvertFunctions( type = type == null ? ValueType.STRING : type; switch (type) { case STRING: - functions[i] = input -> input == null ? "" : input.toString(); + functions[i] = input -> DimensionHandlerUtils.convertObjectToString(input); break; case LONG: - functions[i] = input -> { - final Long val = DimensionHandlerUtils.convertObjectToLong(input); - return val == null ? 0L : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToLong(input); break; case FLOAT: - functions[i] = input -> { - final Float val = DimensionHandlerUtils.convertObjectToFloat(input); - return val == null ? 0.f : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToFloat(input); break; case DOUBLE: - functions[i] = input -> { - Double val = DimensionHandlerUtils.convertObjectToDouble(input); - return val == null ? 0.0 : val; - }; + functions[i] = input -> DimensionHandlerUtils.convertObjectToDouble(input); break; default: throw new IAE("invalid type: [%s]", type); @@ -874,7 +859,10 @@ public int compare(Grouper.Entry entry1, Grouper.Entry private static int compareDimsInRows(RowBasedKey key1, RowBasedKey key2, int dimStart) { for (int i = dimStart; i < key1.getKey().length; i++) { - final int cmp = ((Comparable) key1.getKey()[i]).compareTo(key2.getKey()[i]); + final int cmp = Comparators.naturalNullsFirst().compare( + (Comparable) key1.getKey()[i], + (Comparable) key2.getKey()[i] + ); if (cmp != 0) { return cmp; } @@ -923,9 +911,10 @@ private static int compareDimsInRowsWithAggs( if (isNumericField.get(i) && comparator.equals(StringComparators.NUMERIC)) { // use natural comparison - cmp = lhs.compareTo(rhs); + cmp = Comparators.naturalNullsFirst().compare(lhs, rhs); } else { - cmp = comparator.compare(lhs.toString(), rhs.toString()); + cmp = comparator.compare(DimensionHandlerUtils.convertObjectToString(lhs), + DimensionHandlerUtils.convertObjectToString(rhs)); } if (cmp != 0) { @@ -939,7 +928,8 @@ private static int compareDimsInRowsWithAggs( static long estimateStringKeySize(String key) { - return (long) key.length() * Chars.BYTES + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; + long length = key == null ? 0 : key.length(); + return length * Chars.BYTES + ROUGH_OVERHEAD_PER_DICTIONARY_ENTRY; } private static class RowBasedKeySerde implements Grouper.KeySerde @@ -1027,7 +1017,7 @@ private void initializeRankOfDictionaryIds() rankOfDictionaryIds = IntStream.range(0, dictionarySize).toArray(); IntArrays.quickSort( rankOfDictionaryIds, - (i1, i2) -> dictionary.get(i1).compareTo(dictionary.get(i2)) + (i1, i2) -> Comparators.naturalNullsFirst().compare(dictionary.get(i1), dictionary.get(i2)) ); IntArrayUtils.inverse(rankOfDictionaryIds); @@ -1578,20 +1568,36 @@ private class LongRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Longs.BYTES; + return Longs.BYTES + Byte.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putLong((Long) key.getKey()[idx]); + Long aLong = (Long) key.getKey()[idx]; + if (aLong == null) { + keyBuffer.put((byte) 1); + keyBuffer.putLong(0L); + } else { + keyBuffer.put((byte) 0); + keyBuffer.putLong(aLong); + } return true; } + protected Long readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandling.defaultLongValue(); + } else { + return buffer.getLong(initialOffset + keyBufferPosition + Byte.BYTES); + } + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getLong(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override @@ -1609,7 +1615,8 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper FloatRowBasedKeySerdeHelper( int keyBufferPosition, boolean pushLimitDown, - @Nullable StringComparator stringComparator) + @Nullable StringComparator stringComparator + ) { this.keyBufferPosition = keyBufferPosition; if (isPrimitiveComparable(pushLimitDown, stringComparator)) { @@ -1629,20 +1636,36 @@ private class FloatRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Floats.BYTES; + return Floats.BYTES + Byte.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putFloat((Float) key.getKey()[idx]); + Float aFloat = (Float) key.getKey()[idx]; + if (aFloat == null) { + keyBuffer.put((byte) 1); + keyBuffer.putFloat(0F); + } else { + keyBuffer.put((byte) 0); + keyBuffer.putFloat(aFloat); + } return true; } + protected Float readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandling.defaultFloatValue(); + } else { + return buffer.getFloat(initialOffset + keyBufferPosition + Byte.BYTES); + } + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getFloat(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override @@ -1681,20 +1704,36 @@ private class DoubleRowBasedKeySerdeHelper implements RowBasedKeySerdeHelper @Override public int getKeyBufferValueSize() { - return Doubles.BYTES; + return Doubles.BYTES + Byte.BYTES; } @Override public boolean putToKeyBuffer(RowBasedKey key, int idx) { - keyBuffer.putDouble((Double) key.getKey()[idx]); + Double aDouble = (Double) key.getKey()[idx]; + if (aDouble == null) { + keyBuffer.put((byte) 1); + keyBuffer.putDouble(0.0D); + } else { + keyBuffer.put((byte) 0); + keyBuffer.putDouble(aDouble); + } return true; } + protected Double readFromBuffer(ByteBuffer buffer, int initialOffset) + { + if (buffer.get(initialOffset + keyBufferPosition) == (byte) 1) { + return NullHandling.defaultDoubleValue(); + } else { + return buffer.getDouble(initialOffset + keyBufferPosition + Byte.BYTES); + } + } + @Override public void getFromByteBuffer(ByteBuffer buffer, int initialOffset, int dimValIdx, Comparable[] dimValues) { - dimValues[dimValIdx] = buffer.getDouble(initialOffset + keyBufferPosition); + dimValues[dimValIdx] = readFromBuffer(buffer, initialOffset); } @Override diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedKeySerdeHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedKeySerdeHelper.java index c7e3437ded97..4ee790845f57 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedKeySerdeHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedKeySerdeHelper.java @@ -34,7 +34,7 @@ interface RowBasedKeySerdeHelper /** * Read a value from RowBasedKey at `idx` and put the value at the current position of RowBasedKeySerde's keyBuffer. * advancing the position by the size returned by getKeyBufferValueSize(). - * + *

* If an internal resource limit has been reached and the value could not be added to the keyBuffer, * (e.g., maximum dictionary size exceeded for Strings), this method returns false. * @@ -48,7 +48,7 @@ interface RowBasedKeySerdeHelper /** * Read a value from a ByteBuffer containing a grouping key in the same format as RowBasedKeySerde's keyBuffer and * put the value in `dimValues` at `dimValIdx`. - * + *

* The value to be read resides in the buffer at position (`initialOffset` + the SerdeHelper's keyBufferPosition). * * @param buffer ByteBuffer containing an array of grouping keys for a row diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouper.java index f2faca0d2f69..2e376de26943 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouper.java @@ -39,11 +39,11 @@ /** * A streaming grouper which can aggregate sorted inputs. This grouper can aggregate while its iterator is being * consumed. The aggregation thread and the iterating thread can be different. - * + *

* This grouper is backed by an off-heap circular array. The reading thread is able to read data from an array slot * only when aggregation for the grouping key correspoing to that slot is finished. Since the reading and writing * threads cannot access the same array slot at the same time, they can read/write data without contention. - * + *

* This class uses the spinlock for waiting for at least one slot to become available when the array is empty or full. * If the array is empty, the reading thread waits for the aggregation for an array slot is finished. If the array is * full, the writing thread waits for the reading thread to read at least one aggregate from the array. @@ -88,7 +88,7 @@ public class StreamingMergeSortedGrouper implements Grouper * initial value is -1 which means any data are not written yet. Since it's assumed that the input is sorted by the * grouping key, this variable is moved to the next slot whenever a new grouping key is found. Once it reaches the * last slot of the array, it moves to the first slot. - * + *

* This is always moved ahead of {@link #nextReadIndex}. If the array is full, this variable * cannot be moved until {@link #nextReadIndex} is moved. See {@link #increaseWriteIndex()} for more details. This * variable is always incremented by the writing thread and read by both the writing and the reading threads. @@ -99,7 +99,7 @@ public class StreamingMergeSortedGrouper implements Grouper * Next read index of the array. This points to the array slot which the reading thread will read next. Its initial * value is -1 which means any data are not read yet. This variable can point an array slot only when the aggregation * for that slot is finished. Once it reaches the last slot of the array, it moves to the first slot. - * + *

* This always follows {@link #curWriteIndex}. If the array is empty, this variable cannot be moved until the * aggregation for at least one grouping key is finished which in turn {@link #curWriteIndex} is moved. See * {@link #iterator()} for more details. This variable is always incremented by the reading thread and read by both diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java index db58c36e0579..f5bf10fbe6de 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/DictionaryBuildingStringGroupByColumnSelectorStrategy.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.data.ArrayBasedIndexedInts; @@ -59,7 +60,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, value ); } else { - resultMap.put(selectorPlus.getOutputName(), ""); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultStringValue()); } } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java index 9cabaa5aa1c7..a5261236e59c 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/column/StringGroupByColumnSelectorStrategy.java @@ -21,6 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DimensionSelector; import io.druid.segment.data.IndexedInts; @@ -48,7 +49,7 @@ public void processValueFromGroupingKey(GroupByColumnSelectorPlus selectorPlus, ((DimensionSelector) selectorPlus.getSelector()).lookupName(id) ); } else { - resultMap.put(selectorPlus.getOutputName(), ""); + resultMap.put(selectorPlus.getOutputName(), NullHandling.defaultStringValue()); } } diff --git a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java index 2e382c4dd9cd..7f1127d92519 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java @@ -68,6 +68,10 @@ public void setAggregators(Map aggregators) @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return metricVal == null && value == null; + } return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) == 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java index 1c7ec437eecd..06fe975aea6d 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java @@ -68,6 +68,10 @@ public void setAggregators(Map aggregators) @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return false; + } return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) > 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java b/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java index ff08202bf7cb..e262914e5107 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java +++ b/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java @@ -24,8 +24,8 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import io.druid.data.input.Row; -import io.druid.query.aggregation.AggregatorFactory; import io.druid.java.util.common.ISE; +import io.druid.query.aggregation.AggregatorFactory; import java.util.Map; import java.util.regex.Pattern; diff --git a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java index 37d6268b866d..6ab9747639c2 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java @@ -66,6 +66,10 @@ public void setAggregators(Map aggregators) @Override public boolean eval(Row row) { + Object metricVal = row.getRaw(aggregationName); + if (metricVal == null || value == null) { + return false; + } return HavingSpecMetricComparator.compare(row, aggregationName, value, aggregators) < 0; } diff --git a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java index 5ad75004b760..539fb7b39d63 100644 --- a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java +++ b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java @@ -228,6 +228,14 @@ protected BinaryFn createMergeFn(Query queryParam) // Fudge timestamp, maybe. final DateTime fudgeTimestamp = getUniversalTimestamp(query); + ImmutableMap.Builder context = ImmutableMap.builder(); + context.put("finalize", false); + context.put(GroupByQueryConfig.CTX_KEY_STRATEGY, GroupByStrategySelector.STRATEGY_V2); + if (fudgeTimestamp != null) { + context.put(CTX_KEY_FUDGE_TIMESTAMP, String.valueOf(fudgeTimestamp.getMillis())); + } + context.put(CTX_KEY_OUTERMOST, false); + context.put(GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown()); final GroupByQuery newQuery = new GroupByQuery( query.getDataSource(), @@ -243,14 +251,7 @@ protected BinaryFn createMergeFn(Query queryParam) query.getLimitSpec(), query.getContext() ).withOverriddenContext( - ImmutableMap.of( - "finalize", false, - GroupByQueryConfig.CTX_KEY_STRATEGY, GroupByStrategySelector.STRATEGY_V2, - CTX_KEY_FUDGE_TIMESTAMP, fudgeTimestamp == null ? "" : String.valueOf(fudgeTimestamp.getMillis()), - CTX_KEY_OUTERMOST, false, - // the having spec shouldn't be passed down, so we need to convey the existing limit push down status - GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, query.isApplyLimitPushDown() - ) + context.build() ); Sequence rowSequence = Sequences.map( diff --git a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java index 810afa9d3d2e..f5fef048e94f 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupConfig.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupConfig.java @@ -56,7 +56,9 @@ public LookupConfig( @JsonProperty("snapshotWorkingDir") String snapshotWorkingDir ) { + //CHECKSTYLE.OFF: Regexp this.snapshotWorkingDir = Strings.nullToEmpty(snapshotWorkingDir); + //CHECKSTYLE.ON: Regexp } public String getSnapshotWorkingDir() diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index b0c65a7e098f..54ea60353bc1 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.base.Throwables; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.query.extraction.ExtractionCacheHelper; import io.druid.query.extraction.FunctionalExtraction; @@ -56,9 +56,9 @@ public LookupExtractionFn( { @Nullable @Override - public String apply(String input) + public String apply(@Nullable String input) { - return lookup.apply(Strings.nullToEmpty(input)); + return NullHandling.emptyToNullIfNeeded(lookup.apply(NullHandling.nullToEmptyIfNeeded(input))); } }, retainMissingValue, @@ -120,6 +120,7 @@ public byte[] getCacheKey() outputStream.write(isInjective() ? 1 : 0); outputStream.write(isRetainMissingValue() ? 1 : 0); outputStream.write(isOptimize() ? 1 : 0); + outputStream.write(getReplaceMissingValueWith() == null ? 1 : 0); return outputStream.toByteArray(); } catch (IOException ex) { diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java index 0400c21ae1e6..1a487fab63d5 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractor.java @@ -25,7 +25,6 @@ import io.druid.query.extraction.MapLookupExtractor; import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -45,7 +44,7 @@ public abstract class LookupExtractor * @return The lookup, or null key cannot have the lookup applied to it and should be treated as missing. */ @Nullable - public abstract String apply(@NotNull String key); + public abstract String apply(@Nullable String key); /** * @param keys set of keys to apply lookup for each element diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 36c36d256b46..8743d106a900 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -21,12 +21,12 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Accumulator; @@ -218,8 +218,8 @@ private ColumnAnalysis analyzeStringColumn( } if (analyzingMinMax() && cardinality > 0) { - min = Strings.nullToEmpty(bitmapIndex.getValue(0)); - max = Strings.nullToEmpty(bitmapIndex.getValue(cardinality - 1)); + min = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(0)); + max = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(cardinality - 1)); } return new ColumnAnalysis( diff --git a/processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java b/processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java index 9b743aa81cca..90ff28b07c96 100644 --- a/processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java @@ -37,5 +37,6 @@ public interface ColumnIncluderator byte[] LIST_CACHE_PREFIX = new byte[]{0x2}; boolean include(String columnName); + byte[] getCacheKey(); } diff --git a/processing/src/main/java/io/druid/query/search/SearchHit.java b/processing/src/main/java/io/druid/query/search/SearchHit.java index 52e6692f989a..4031d0df3d88 100644 --- a/processing/src/main/java/io/druid/query/search/SearchHit.java +++ b/processing/src/main/java/io/druid/query/search/SearchHit.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import io.druid.common.config.NullHandling; /** */ @@ -39,7 +40,7 @@ public SearchHit( ) { this.dimension = Preconditions.checkNotNull(dimension); - this.value = Preconditions.checkNotNull(value); + this.value = NullHandling.nullToEmptyIfNeeded(value); this.count = count; } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 7921d717b9a4..890d8c7ec328 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -20,7 +20,6 @@ package io.druid.query.search; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -131,7 +130,7 @@ public void updateSearchResultSet( for (int i = 0; i < vals.size(); ++i) { final String dimVal = selector.lookupName(vals.get(i)); if (searchQuerySpec.accept(dimVal)) { - set.addTo(new SearchHit(outputName, Strings.nullToEmpty(dimVal)), 1); + set.addTo(new SearchHit(outputName, dimVal), 1); if (set.size() >= limit) { return; } diff --git a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java index ec4ce2078546..987a86e0d685 100644 --- a/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java +++ b/processing/src/main/java/io/druid/query/search/UseIndexesStrategy.java @@ -20,7 +20,6 @@ package io.druid.query.search; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.collections.bitmap.BitmapFactory; @@ -261,7 +260,7 @@ public Object2IntRBTreeMap execute(int limit) extractionFn = IdentityExtractionFn.getInstance(); } for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + String dimVal = extractionFn.apply(bitmapIndex.getValue(i)); if (!searchQuerySpec.accept(dimVal)) { continue; } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index bd3268f003c1..a95c7eabf158 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -166,6 +166,7 @@ public void addRowValuesToSelectResult( } } } + public static class DoubleSelectColumnSelectorStrategy implements SelectColumnSelectorStrategy { diff --git a/processing/src/main/java/io/druid/query/topn/TopNMapFn.java b/processing/src/main/java/io/druid/query/topn/TopNMapFn.java index 24d05fa37775..3d4807628ec9 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNMapFn.java +++ b/processing/src/main/java/io/druid/query/topn/TopNMapFn.java @@ -56,14 +56,10 @@ public static Function getValueTransformer(ValueType outputType) return longVal == null ? DimensionHandlerUtils.ZERO_LONG : longVal; }; - private static Function FLOAT_TRANSFORMER = input -> { - final Float floatVal = DimensionHandlerUtils.convertObjectToFloat(input); - return floatVal == null ? DimensionHandlerUtils.ZERO_FLOAT : floatVal; - }; - private static Function DOUBLE_TRANSFORMER = input -> { - final Double doubleValue = DimensionHandlerUtils.convertObjectToDouble(input); - return doubleValue == null ? DimensionHandlerUtils.ZERO_DOUBLE : doubleValue; - }; + private static Function FLOAT_TRANSFORMER = input -> DimensionHandlerUtils.convertObjectToFloat(input); + + private static Function DOUBLE_TRANSFORMER = input -> DimensionHandlerUtils.convertObjectToDouble( + input); private static final TopNColumnSelectorStrategyFactory STRATEGY_FACTORY = new TopNColumnSelectorStrategyFactory(); diff --git a/processing/src/main/java/io/druid/segment/AggregatorNullHandling.java b/processing/src/main/java/io/druid/segment/AggregatorNullHandling.java new file mode 100644 index 000000000000..fe4aefaf3287 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/AggregatorNullHandling.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.base.Supplier; +import io.druid.common.config.NullHandling; + +public class AggregatorNullHandling +{ + + public static Supplier getNullableSupplier( + final Supplier supplier, + final BaseNullableColumnValueSelector selector + ) + { + return NullHandling.useDefaultValuesForNull() ? + supplier : () -> { + if (selector.isNull()) { + return null; + } + return supplier.get(); + }; + } +} diff --git a/processing/src/main/java/io/druid/segment/BaseDoubleColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseDoubleColumnValueSelector.java index e1d0d40d23f7..de6b521620af 100644 --- a/processing/src/main/java/io/druid/segment/BaseDoubleColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseDoubleColumnValueSelector.java @@ -27,11 +27,11 @@ * Double value selecting polymorphic "part" of the {@link ColumnValueSelector} interface. Users of {@link * ColumnValueSelector#getDouble()} are encouraged to reduce the parameter/field/etc. type to * BaseDoubleColumnValueSelector to make it impossible to accidently call any method other than {@link #getDouble()}. - * + *

* All implementations of this interface MUST also implement {@link ColumnValueSelector}. */ @PublicApi -public interface BaseDoubleColumnValueSelector extends HotLoopCallee +public interface BaseDoubleColumnValueSelector extends HotLoopCallee, BaseNullableColumnValueSelector { @CalledFromHotLoop double getDouble(); diff --git a/processing/src/main/java/io/druid/segment/BaseFloatColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseFloatColumnValueSelector.java index 9223ff339422..8ce86e707e95 100644 --- a/processing/src/main/java/io/druid/segment/BaseFloatColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseFloatColumnValueSelector.java @@ -27,12 +27,13 @@ * Float value selecting polymorphic "part" of the {@link ColumnValueSelector} interface. Users of {@link * ColumnValueSelector#getFloat()} are encouraged to reduce the parameter/field/etc. type to * BaseFloatColumnValueSelector to make it impossible to accidently call any method other than {@link #getFloat()}. - * + *

* All implementations of this interface MUST also implement {@link ColumnValueSelector}. */ @PublicApi -public interface BaseFloatColumnValueSelector extends HotLoopCallee +public interface BaseFloatColumnValueSelector extends HotLoopCallee, BaseNullableColumnValueSelector { @CalledFromHotLoop float getFloat(); + } diff --git a/processing/src/main/java/io/druid/segment/BaseLongColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseLongColumnValueSelector.java index ac06ab7d3273..a6bfa438ec6a 100644 --- a/processing/src/main/java/io/druid/segment/BaseLongColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseLongColumnValueSelector.java @@ -27,11 +27,11 @@ * Long value selecting polymorphic "part" of the {@link ColumnValueSelector} interface. Users of {@link * ColumnValueSelector#getLong()} are encouraged to reduce the parameter/field/etc. type to BaseLongColumnValueSelector * to make it impossible to accidently call any method other than {@link #getLong()}. - * + *

* All implementations of this interface MUST also implement {@link ColumnValueSelector}. */ @PublicApi -public interface BaseLongColumnValueSelector extends HotLoopCallee +public interface BaseLongColumnValueSelector extends HotLoopCallee, BaseNullableColumnValueSelector { @CalledFromHotLoop long getLong(); diff --git a/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java new file mode 100644 index 000000000000..3d2c6a5b990f --- /dev/null +++ b/processing/src/main/java/io/druid/segment/BaseNullableColumnValueSelector.java @@ -0,0 +1,28 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.guice.annotations.PublicApi; + +@PublicApi +public interface BaseNullableColumnValueSelector +{ + boolean isNull(); +} diff --git a/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java b/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java index 619e9a23d2cc..1afd14650da7 100644 --- a/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/BaseObjectColumnValueSelector.java @@ -27,7 +27,7 @@ * Object value selecting polymorphic "part" of the {@link ColumnValueSelector} interface. Users of {@link * ColumnValueSelector#getObject()} are encouraged to reduce the parameter/field/etc. type to * BaseObjectColumnValueSelector to make it impossible to accidently call any method other than {@link #getObject()}. - * + *

* All implementations of this interface MUST also implement {@link ColumnValueSelector}. */ @PublicApi diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index ce9dbbf2727b..d333f5900700 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -19,10 +19,10 @@ package io.druid.segment; -import com.google.common.base.Strings; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; +import io.druid.common.config.NullHandling; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.column.BitmapIndex; @@ -180,7 +180,7 @@ public int getIndex(String value) // Return -2 for non-null values to match what the BitmapIndex implementation in BitmapIndexColumnPartSupplier // would return for getIndex() when there is only a single index, for the null value. // i.e., return an 'insertion point' of 1 for non-null values (see BitmapIndex interface) - return Strings.isNullOrEmpty(value) ? 0 : -2; + return value == null ? 0 : -2; } @Override @@ -210,7 +210,7 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) final Column column = index.getColumn(dimension); if (column == null || !columnSupportsFiltering(column)) { - if (Strings.isNullOrEmpty(value)) { + if (NullHandling.isNullOrEquivalent(value)) { return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), getNumRows()); } else { return bitmapFactory.makeEmptyImmutableBitmap(); diff --git a/processing/src/main/java/io/druid/segment/ColumnValueSelector.java b/processing/src/main/java/io/druid/segment/ColumnValueSelector.java index c6a147d16c02..b6128178173a 100644 --- a/processing/src/main/java/io/druid/segment/ColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnValueSelector.java @@ -37,4 +37,5 @@ public interface ColumnValueSelector extends BaseLongColumnValueSelector, BaseDoubleColumnValueSelector, BaseFloatColumnValueSelector, BaseObjectColumnValueSelector { + } diff --git a/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java b/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java index 010dd841dd21..087ce74de2ae 100644 --- a/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/ConstantColumnValueSelector.java @@ -86,4 +86,10 @@ public void inspectRuntimeShape(final RuntimeShapeInspector inspector) { // Nothing here: objectValue is nullable but getObject is not @CalledFromHotLoop } + + @Override + public boolean isNull() + { + return objectValue == null; + } } diff --git a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java index 8210d7d5c948..5f05f8f9d6d5 100644 --- a/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/ConstantDimensionSelector.java @@ -20,7 +20,7 @@ package io.druid.segment; import com.google.common.base.Predicate; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; @@ -37,7 +37,7 @@ public class ConstantDimensionSelector implements SingleValueHistoricalDimension public ConstantDimensionSelector(final String value) { - if (Strings.isNullOrEmpty(value)) { + if (NullHandling.isNullOrEquivalent(value)) { // There's an optimized implementation for nulls that callers should use instead. throw new IllegalArgumentException("Use NullDimensionSelector or DimensionSelectorUtils.constantSelector"); } diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index 504e46c4f97f..c77adca51f8f 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; +import io.druid.common.config.NullHandling; import io.druid.common.guava.GuavaUtils; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; import io.druid.java.util.common.IAE; @@ -124,17 +125,18 @@ public static Colum * Creates an array of ColumnSelectorPlus objects, selectors that handle type-specific operations within * query processing engines, using a strategy factory provided by the query engine. One ColumnSelectorPlus * will be created for each column specified in dimensionSpecs. - * + *

* The ColumnSelectorPlus provides access to a type strategy (e.g., how to group on a float column) * and a value selector for a single column. - * + *

* A caller should define a strategy factory that provides an interface for type-specific operations * in a query engine. See GroupByStrategyFactory for a reference. * * @param The strategy type created by the provided strategy factory. - * @param strategyFactory A factory provided by query engines that generates type-handling strategies - * @param dimensionSpecs The set of columns to generate ColumnSelectorPlus objects for - * @param columnSelectorFactory Used to create value selectors for columns. + * @param strategyFactory A factory provided by query engines that generates type-handling strategies + * @param dimensionSpecs The set of columns to generate ColumnSelectorPlus objects for + * @param columnSelectorFactory Used to create value selectors for columns. + * * @return An array of ColumnSelectorPlus objects, in the order of the columns specified in dimensionSpecs */ public static @@ -237,11 +239,20 @@ private static Colu return strategyFactory.makeColumnSelectorStrategy(capabilities, selector); } + @Nullable + public static String convertObjectToString(@Nullable Object valObj) + { + if (valObj == null) { + return null; + } + return valObj.toString(); + } + @Nullable public static Long convertObjectToLong(@Nullable Object valObj) { if (valObj == null) { - return ZERO_LONG; + return null; } if (valObj instanceof Long) { @@ -259,7 +270,7 @@ public static Long convertObjectToLong(@Nullable Object valObj) public static Float convertObjectToFloat(@Nullable Object valObj) { if (valObj == null) { - return ZERO_FLOAT; + return null; } if (valObj instanceof Float) { @@ -277,7 +288,7 @@ public static Float convertObjectToFloat(@Nullable Object valObj) public static Double convertObjectToDouble(@Nullable Object valObj) { if (valObj == null) { - return ZERO_DOUBLE; + return null; } if (valObj instanceof Double) { @@ -285,7 +296,8 @@ public static Double convertObjectToDouble(@Nullable Object valObj) } else if (valObj instanceof Number) { return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { - return Doubles.tryParse((String) valObj); + Double doubleVal = Doubles.tryParse((String) valObj); + return doubleVal; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } @@ -293,7 +305,7 @@ public static Double convertObjectToDouble(@Nullable Object valObj) /** * Convert a string representing a decimal value to a long. - * + *

* If the decimal value is not an exact integral value (e.g. 42.0), or if the decimal value * is too large to be contained within a long, this function returns null. * @@ -327,18 +339,18 @@ public static Long getExactLongFromDecimalString(String decimalStr) } } - public static Double nullToZero(@Nullable Double number) + public static Number nullToZeroDouble(@Nullable Number number) { - return number == null ? ZERO_DOUBLE : number; + return number == null ? NullHandling.ZERO_DOUBLE : number; } - public static Long nullToZero(@Nullable Long number) + public static Number nullToZeroLong(@Nullable Number number) { - return number == null ? ZERO_LONG : number; + return number == null ? NullHandling.ZERO_LONG : number; } - public static Float nullToZero(@Nullable Float number) + public static Number nullToZeroFloat(@Nullable Number number) { - return number == null ? ZERO_FLOAT : number; + return number == null ? NullHandling.ZERO_FLOAT : number; } } diff --git a/processing/src/main/java/io/druid/segment/DimensionSelector.java b/processing/src/main/java/io/druid/segment/DimensionSelector.java index 91e099070481..940edc50adbe 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelector.java @@ -150,12 +150,34 @@ default long getLong() } /** + * @deprecated always throws {@link UnsupportedOperationException} + */ + @Deprecated + @Override + default boolean isNull() + { + throw new UnsupportedOperationException("DimensionSelector cannot be operated as numeric ColumnValueSelector" + this.getClass()); + } + + /** + * @deprecated always throws {@link UnsupportedOperationException} + */ + @Deprecated + @Override + default Object getObject() + { + throw new UnsupportedOperationException("DimensionSelector cannot be operated as object ColumnValueSelector"); + } + + /** + * @deprecated always throws {@link UnsupportedOperationException} * Converts the current result of {@link #getRow()} into null, if the row is empty, a String, if the row has size 1, * or a String[] array, if the row has size > 1, using {@link #lookupName(int)}. * * This method is not the default implementation of {@link #getObject()} to minimize the chance that implementations * "forget" to override it with more optimized version. */ + @Deprecated @Nullable default Object defaultGetObject() { diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java index 1fb6c53cefbc..4e5a3bbd9b5a 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSelector.java @@ -22,7 +22,7 @@ /** * This interface is convenient for implementation of "double-sourcing" {@link ColumnValueSelector}s, it provides * default implementations for all {@link ColumnValueSelector}'s methods except {@link #getDouble()}. - * + *

* This interface should appear ONLY in "implements" clause or anonymous class creation, but NOT in "user" code, where * {@link BaseDoubleColumnValueSelector} must be used instead. */ @@ -61,6 +61,9 @@ default long getLong() @Override default Double getObject() { + if (isNull()) { + return null; + } return getDouble(); } diff --git a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java index e6b6df6098f8..f94fcf5abe9b 100644 --- a/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/DoubleColumnSerializer.java @@ -19,14 +19,21 @@ package io.druid.segment; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; -import io.druid.segment.writeout.SegmentWriteOutMedium; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.CompressionStrategy; +import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.data.ColumnarDoublesSerializer; +import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -35,29 +42,43 @@ public class DoubleColumnSerializer implements GenericColumnSerializer public static DoubleColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - CompressionStrategy compression + CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { - return new DoubleColumnSerializer(segmentWriteOutMedium, filenameBase, IndexIO.BYTE_ORDER, compression); + return new DoubleColumnSerializer( + segmentWriteOutMedium, + filenameBase, + IndexIO.BYTE_ORDER, + compression, + bitmapSerdeFactory + ); } private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ByteOrder byteOrder; private final CompressionStrategy compression; + private final BitmapSerdeFactory bitmapSerdeFactory; + private ColumnarDoublesSerializer writer; + private ByteBufferWriter nullValueBitmapWriter; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; private DoubleColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, - CompressionStrategy compression + CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; + this.bitmapSerdeFactory = bitmapSerdeFactory; } @Override @@ -70,25 +91,44 @@ public void open() throws IOException compression ); writer.open(); + nullValueBitmapWriter = new ByteBufferWriter<>( + segmentWriteOutMedium, + bitmapSerdeFactory.getObjectStrategy() + ); + nullValueBitmapWriter.open(); + nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); } @Override - public void serialize(Object obj) throws IOException + public void serialize(@Nullable Object obj) throws IOException { - double val = (obj == null) ? 0 : ((Number) obj).doubleValue(); - writer.add(val); + if (obj == null) { + nullRowsBitmap.add(rowCount); + writer.add(0D); + } else { + writer.add(((Number) obj).doubleValue()); + } + rowCount++; } @Override public long getSerializedSize() throws IOException { - return writer.getSerializedSize(); + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); + long bitmapSize = nullRowsBitmap.isEmpty() + ? 0L + : nullValueBitmapWriter.getSerializedSize(); + return Integer.BYTES + writer.getSerializedSize() + bitmapSize; } @Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); writer.writeTo(channel, smoosher); + if (!nullRowsBitmap.isEmpty()) { + nullValueBitmapWriter.writeTo(channel, smoosher); + } } } diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index 26d71abc2933..e4b43e7fbf38 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -21,6 +21,8 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.common.config.NullHandling; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.Indexed; @@ -28,10 +30,13 @@ import io.druid.segment.incremental.TimeAndDimsHolder; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; +import java.util.Objects; public class DoubleDimensionIndexer implements DimensionIndexer { + public static final Comparator DOUBLE_COMPARATOR = Comparators.naturalNullsFirst(); @Override public Double processRowValsToUnsortedEncodedKeyComponent(Object dimValues) @@ -91,12 +96,23 @@ public ColumnValueSelector makeColumnValueSelector( final int dimIndex = desc.getIndex(); class IndexerDoubleColumnSelector implements DoubleColumnSelector { + + @Override + public boolean isNull() + { + if (NullHandling.useDefaultValuesForNull()) { + return false; + } + final Object[] dims = currEntry.get().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; + } + @Override public double getDouble() { final Object[] dims = currEntry.get().getDims(); - if (dimIndex >= dims.length) { + if (dimIndex >= dims.length || dims[dimIndex] == null) { return 0.0; } return (Double) dims[dimIndex]; @@ -112,6 +128,7 @@ public Double getObject() if (dimIndex >= dims.length) { return null; } + return (Double) dims[dimIndex]; } @@ -128,19 +145,19 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public int compareUnsortedEncodedKeyComponents(@Nullable Double lhs, @Nullable Double rhs) { - return Double.compare(DimensionHandlerUtils.nullToZero(lhs), DimensionHandlerUtils.nullToZero(rhs)); + return DOUBLE_COMPARATOR.compare(lhs, rhs); } @Override public boolean checkUnsortedEncodedKeyComponentsEqual(@Nullable Double lhs, @Nullable Double rhs) { - return DimensionHandlerUtils.nullToZero(lhs).equals(DimensionHandlerUtils.nullToZero(rhs)); + return Objects.equals(lhs, rhs); } @Override public int getUnsortedEncodedKeyComponentHashCode(@Nullable Double key) { - return DimensionHandlerUtils.nullToZero(key).hashCode(); + return DimensionHandlerUtils.nullToZeroDouble(key).hashCode(); } @Override diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java index 3702df043127..7d87798de8f8 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionMergerV9.java @@ -22,7 +22,7 @@ import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressionStrategy; -import io.druid.segment.serde.DoubleGenericColumnPartSerde; +import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; @@ -55,7 +55,12 @@ public DoubleDimensionMergerV9( private void setupEncodedValueWriter(SegmentWriteOutMedium segmentWriteOutMedium) throws IOException { final CompressionStrategy metCompression = indexSpec.getMetricCompression(); - this.serializer = DoubleColumnSerializer.create(segmentWriteOutMedium, dimensionName, metCompression); + this.serializer = DoubleColumnSerializer.create( + segmentWriteOutMedium, + dimensionName, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); serializer.open(); } @@ -80,13 +85,12 @@ public void processMergedRow(Double rowValues) throws IOException @Override public void writeIndexes(List segmentRowNumConversions) throws IOException { - // double columns do not have indexes + // doubles have no indices to write } @Override public boolean canSkip() { - // a double column can never be all null return false; } @@ -96,10 +100,11 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.DOUBLE); builder.addSerde( - DoubleGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .build() + DoubleGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate(serializer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .build() ); return builder.build(); } diff --git a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java index 695d20871cad..1e41305160a8 100644 --- a/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSerializer.java @@ -19,14 +19,21 @@ package io.druid.segment; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ColumnarFloatsSerializer; +import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -35,29 +42,37 @@ public class FloatColumnSerializer implements GenericColumnSerializer public static FloatColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, - CompressionStrategy compression + CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { - return new FloatColumnSerializer(segmentWriteOutMedium, filenameBase, IndexIO.BYTE_ORDER, compression); + return new FloatColumnSerializer(segmentWriteOutMedium, filenameBase, IndexIO.BYTE_ORDER, compression, bitmapSerdeFactory); } private final SegmentWriteOutMedium segmentWriteOutMedium; private final String filenameBase; private final ByteOrder byteOrder; private final CompressionStrategy compression; + private final BitmapSerdeFactory bitmapSerdeFactory; + private ColumnarFloatsSerializer writer; + private ByteBufferWriter nullValueBitmapWriter; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; private FloatColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, - CompressionStrategy compression + CompressionStrategy compression, + BitmapSerdeFactory bitmapSerdeFactory ) { this.segmentWriteOutMedium = segmentWriteOutMedium; this.filenameBase = filenameBase; this.byteOrder = byteOrder; this.compression = compression; + this.bitmapSerdeFactory = bitmapSerdeFactory; } @Override @@ -70,24 +85,44 @@ public void open() throws IOException compression ); writer.open(); + nullValueBitmapWriter = new ByteBufferWriter<>( + segmentWriteOutMedium, + bitmapSerdeFactory.getObjectStrategy() + ); + nullValueBitmapWriter.open(); + nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); } @Override - public void serialize(Object obj) throws IOException + public void serialize(@Nullable Object obj) throws IOException { - float val = (obj == null) ? 0 : ((Number) obj).floatValue(); - writer.add(val); + if (obj == null) { + nullRowsBitmap.add(rowCount); + writer.add(0L); + } else { + writer.add(((Number) obj).floatValue()); + } + rowCount++; } @Override + public long getSerializedSize() throws IOException { - return writer.getSerializedSize(); + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); + long bitmapSize = nullRowsBitmap.isEmpty() + ? 0L + : nullValueBitmapWriter.getSerializedSize(); + return Integer.BYTES + writer.getSerializedSize() + bitmapSize; } @Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); writer.writeTo(channel, smoosher); + if (!nullRowsBitmap.isEmpty()) { + nullValueBitmapWriter.writeTo(channel, smoosher); + } } } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index 49c93f576107..b8f8e22e78b1 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -21,6 +21,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.Indexed; @@ -28,11 +29,15 @@ import io.druid.segment.incremental.TimeAndDimsHolder; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; +import java.util.Objects; public class FloatDimensionIndexer implements DimensionIndexer { + public static final Comparator FLOAT_COMPARATOR = Comparators.naturalNullsFirst(); + @Override public Float processRowValsToUnsortedEncodedKeyComponent(Object dimValues) { @@ -97,13 +102,20 @@ public float getFloat() { final Object[] dims = currEntry.get().getDims(); - if (dimIndex >= dims.length) { + if (dimIndex >= dims.length || dims[dimIndex] == null) { return 0.0f; } return (Float) dims[dimIndex]; } + @Override + public boolean isNull() + { + final Object[] dims = currEntry.get().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; + } + @SuppressWarnings("deprecation") @Nullable @Override @@ -131,19 +143,19 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public int compareUnsortedEncodedKeyComponents(@Nullable Float lhs, @Nullable Float rhs) { - return DimensionHandlerUtils.nullToZero(lhs).compareTo(DimensionHandlerUtils.nullToZero(rhs)); + return FLOAT_COMPARATOR.compare(lhs, rhs); } @Override public boolean checkUnsortedEncodedKeyComponentsEqual(@Nullable Float lhs, @Nullable Float rhs) { - return DimensionHandlerUtils.nullToZero(lhs).equals(DimensionHandlerUtils.nullToZero(rhs)); + return Objects.equals(lhs, rhs); } @Override public int getUnsortedEncodedKeyComponentHashCode(@Nullable Float key) { - return DimensionHandlerUtils.nullToZero(key).hashCode(); + return DimensionHandlerUtils.nullToZeroFloat(key).hashCode(); } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java index 0a2070f4cf1f..f1d41de12116 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionMergerV9.java @@ -21,8 +21,8 @@ import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; +import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; import io.druid.segment.data.CompressionStrategy; -import io.druid.segment.serde.FloatGenericColumnPartSerde; import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; @@ -43,7 +43,6 @@ public FloatDimensionMergerV9( { this.dimensionName = dimensionName; this.indexSpec = indexSpec; - try { setupEncodedValueWriter(segmentWriteOutMedium); } @@ -55,7 +54,12 @@ public FloatDimensionMergerV9( private void setupEncodedValueWriter(SegmentWriteOutMedium segmentWriteOutMedium) throws IOException { final CompressionStrategy metCompression = indexSpec.getMetricCompression(); - this.serializer = FloatColumnSerializer.create(segmentWriteOutMedium, dimensionName, metCompression); + this.serializer = FloatColumnSerializer.create( + segmentWriteOutMedium, + dimensionName, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); serializer.open(); } @@ -86,7 +90,6 @@ public void writeIndexes(List segmentRowNumConversions) throws IOExce @Override public boolean canSkip() { - // a float column can never be all null return false; } @@ -96,10 +99,11 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.FLOAT); builder.addSerde( - FloatGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .build() + FloatGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .withDelegate(serializer) + .build() ); return builder.build(); } diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 6a13e7bf0f87..fe19a3bff75f 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -84,6 +84,7 @@ public class IndexIO public static final byte V8_VERSION = 0x8; public static final byte V9_VERSION = 0x9; public static final int CURRENT_VERSION_ID = V9_VERSION; + public static BitmapSerdeFactory LEGACY_FACTORY = new BitmapSerde.LegacyBitmapSerdeFactory(); public static final ByteOrder BYTE_ORDER = ByteOrder.nativeOrder(); @@ -162,13 +163,11 @@ public void validateTwoSegments(final IndexableAdapter adapter1, final Indexable if (rb1.getRowNum() != rb2.getRowNum()) { throw new SegmentValidationException("Row number mismatch: [%d] vs [%d]", rb1.getRowNum(), rb2.getRowNum()); } - if (rb1.compareTo(rb2) != 0) { - try { - validateRowValues(dimHandlers, rb1, adapter1, rb2, adapter2); - } - catch (SegmentValidationException ex) { - throw new SegmentValidationException(ex, "Validation failure on row %d: [%s] vs [%s]", row, rb1, rb2); - } + try { + validateRowValues(dimHandlers, rb1, adapter1, rb2, adapter2); + } + catch (SegmentValidationException ex) { + throw new SegmentValidationException(ex, "Validation failure on row %d: [%s] vs [%s]", row, rb1, rb2); } } if (it2.hasNext()) { @@ -477,7 +476,11 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException metric, new ColumnBuilder() .setType(ValueType.FLOAT) - .setGenericColumn(new FloatGenericColumnSupplier(metricHolder.floatType)) + .setGenericColumn(new FloatGenericColumnSupplier( + metricHolder.floatType, + LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )) .build() ); } else if (metricHolder.getType() == MetricHolder.MetricType.COMPLEX) { @@ -507,7 +510,11 @@ public QueryableIndex load(File inDir, ObjectMapper mapper) throws IOException Column.TIME_COLUMN_NAME, new ColumnBuilder() .setType(ValueType.LONG) - .setGenericColumn(new LongGenericColumnSupplier(index.timestamps)) + .setGenericColumn(new LongGenericColumnSupplier( + index.timestamps, + LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )) .build() ); return new SimpleQueryableIndex( diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index 858658987b1f..ede5f5ee4b93 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -21,14 +21,15 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; import com.google.common.collect.PeekingIterator; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.inject.ImplementedBy; +import io.druid.common.config.NullHandling; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.ByteBufferUtils; import io.druid.java.util.common.ISE; @@ -53,10 +54,10 @@ import java.nio.IntBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.PriorityQueue; import java.util.Set; import java.util.stream.Collectors; @@ -432,16 +433,10 @@ class DictionaryMergeIterator implements CloseableIterator DictionaryMergeIterator(Indexed[] dimValueLookups, boolean useDirect) { + final Ordering stringOrdering = Comparators.naturalNullsFirst(); pQueue = new PriorityQueue<>( dimValueLookups.length, - new Comparator>>() - { - @Override - public int compare(Pair> lhs, Pair> rhs) - { - return lhs.rhs.peek().compareTo(rhs.rhs.peek()); - } - } + (lhs, rhs) -> stringOrdering.compare(lhs.rhs.peek(), rhs.rhs.peek()) ); conversions = new IntBuffer[dimValueLookups.length]; for (int i = 0; i < conversions.length; i++) { @@ -462,14 +457,7 @@ public int compare(Pair> lhs, Pair iter = Iterators.peekingIterator( Iterators.transform( indexed.iterator(), - new Function() - { - @Override - public String apply(@Nullable String input) - { - return Strings.nullToEmpty(input); - } - } + input -> NullHandling.nullToEmptyIfNeeded(input) ) ); if (iter.hasNext()) { @@ -493,7 +481,7 @@ public String next() } final String value = writeTranslate(smallest, counter); - while (!pQueue.isEmpty() && value.equals(pQueue.peek().rhs.peek())) { + while (!pQueue.isEmpty() && Objects.equals(value, pQueue.peek().rhs.peek())) { writeTranslate(pQueue.remove(), counter); } counter++; diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index 6d9be2ced61c..0d2578063b5f 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -61,9 +61,9 @@ import io.druid.segment.serde.ComplexColumnPartSerde; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; -import io.druid.segment.serde.DoubleGenericColumnPartSerde; -import io.druid.segment.serde.FloatGenericColumnPartSerde; -import io.druid.segment.serde.LongGenericColumnPartSerde; +import io.druid.segment.serde.DoubleGenericColumnPartSerdeV2; +import io.druid.segment.serde.FloatGenericColumnPartSerdeV2; +import io.druid.segment.serde.LongGenericColumnPartSerdeV2; import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; @@ -95,7 +95,11 @@ public class IndexMergerV9 implements IndexMerger private final SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory; @Inject - public IndexMergerV9(ObjectMapper mapper, IndexIO indexIO, SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory) + public IndexMergerV9( + ObjectMapper mapper, + IndexIO indexIO, + SegmentWriteOutMediumFactory defaultSegmentWriteOutMediumFactory + ) { this.mapper = Preconditions.checkNotNull(mapper, "null ObjectMapper"); this.indexIO = Preconditions.checkNotNull(indexIO, "null IndexIO"); @@ -212,7 +216,15 @@ public Metadata apply(IndexableAdapter input) final String section = "build inverted index and columns"; progress.startSection(section); makeTimeColumn(v9Smoosher, progress, timeWriter); - makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsValueTypes, metricTypeNames, metWriters); + makeMetricsColumns( + v9Smoosher, + progress, + mergedMetrics, + metricsValueTypes, + metricTypeNames, + metWriters, + indexSpec + ); for (int i = 0; i < mergedDimensions.size(); i++) { DimensionMergerV9 merger = (DimensionMergerV9) mergers.get(i); @@ -323,7 +335,8 @@ private void makeMetricsColumns( final List mergedMetrics, final Map metricsValueTypes, final Map metricTypeNames, - final List metWriters + final List metWriters, + final IndexSpec indexSpec ) throws IOException { final String section = "make metric columns"; @@ -341,30 +354,33 @@ private void makeMetricsColumns( case LONG: builder.setValueType(ValueType.LONG); builder.addSerde( - LongGenericColumnPartSerde + LongGenericColumnPartSerdeV2 .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((LongColumnSerializer) writer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .build() ); break; case FLOAT: builder.setValueType(ValueType.FLOAT); builder.addSerde( - FloatGenericColumnPartSerde + FloatGenericColumnPartSerdeV2 .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((FloatColumnSerializer) writer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .build() ); break; case DOUBLE: builder.setValueType(ValueType.DOUBLE); builder.addSerde( - DoubleGenericColumnPartSerde + DoubleGenericColumnPartSerdeV2 .serializerBuilder() .withByteOrder(IndexIO.BYTE_ORDER) .withDelegate((DoubleColumnSerializer) writer) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) .build() ); break; @@ -403,10 +419,10 @@ private void makeTimeColumn( .builder() .setValueType(ValueType.LONG) .addSerde( - LongGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(timeWriter) - .build() + LongGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate(timeWriter) + .build() ) .build(); makeColumn(v9Smoosher, Column.TIME_COLUMN_NAME, serdeficator); @@ -471,7 +487,9 @@ private void mergeIndexesAndWriteColumns( merger.processMergedRow(dims[i]); } - Iterator> rowsIterator = theRow.getComprisedRows().int2ObjectEntrySet().fastIterator(); + Iterator> rowsIterator = theRow.getComprisedRows() + .int2ObjectEntrySet() + .fastIterator(); while (rowsIterator.hasNext()) { Int2ObjectMap.Entry comprisedRow = rowsIterator.next(); @@ -498,13 +516,13 @@ private void mergeIndexesAndWriteColumns( progress.stopSection(section); } - private LongColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) throws IOException + private LongColumnSerializer setupTimeWriter(SegmentWriteOutMedium segmentWriteOutMedium, IndexSpec indexSpec) + throws IOException { LongColumnSerializer timeWriter = LongColumnSerializer.create( - segmentWriteOutMedium, - "little_end_time", - CompressionStrategy.DEFAULT_COMPRESSION_STRATEGY, - indexSpec.getLongEncoding() + segmentWriteOutMedium, "little_end_time", CompressionStrategy.DEFAULT_COMPRESSION_STRATEGY, + indexSpec.getLongEncoding(), + indexSpec.getBitmapSerdeFactory() ); // we will close this writer after we added all the timestamps timeWriter.open(); @@ -527,13 +545,29 @@ private ArrayList setupMetricsWriters( GenericColumnSerializer writer; switch (type) { case LONG: - writer = LongColumnSerializer.create(segmentWriteOutMedium, metric, metCompression, longEncoding); + writer = LongColumnSerializer.create( + segmentWriteOutMedium, + metric, + metCompression, + longEncoding, + indexSpec.getBitmapSerdeFactory() + ); break; case FLOAT: - writer = FloatColumnSerializer.create(segmentWriteOutMedium, metric, metCompression); + writer = FloatColumnSerializer.create( + segmentWriteOutMedium, + metric, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); break; case DOUBLE: - writer = DoubleColumnSerializer.create(segmentWriteOutMedium, metric, metCompression); + writer = DoubleColumnSerializer.create( + segmentWriteOutMedium, + metric, + metCompression, + indexSpec.getBitmapSerdeFactory() + ); break; case COMPLEX: final String typeName = metricTypeNames.get(metric); diff --git a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java index bf8765680716..605d2aea23e1 100644 --- a/processing/src/main/java/io/druid/segment/LongColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongColumnSerializer.java @@ -19,14 +19,21 @@ package io.druid.segment; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.writeout.SegmentWriteOutMedium; import io.druid.segment.data.CompressionFactory; import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ColumnarLongsSerializer; +import javax.annotation.Nullable; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.WritableByteChannel; @@ -39,10 +46,18 @@ public static LongColumnSerializer create( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, CompressionStrategy compression, - CompressionFactory.LongEncodingStrategy encoding + CompressionFactory.LongEncodingStrategy encoding, + BitmapSerdeFactory bitmapSerdeFactory ) { - return new LongColumnSerializer(segmentWriteOutMedium, filenameBase, IndexIO.BYTE_ORDER, compression, encoding); + return new LongColumnSerializer( + segmentWriteOutMedium, + filenameBase, + IndexIO.BYTE_ORDER, + compression, + encoding, + bitmapSerdeFactory + ); } private final SegmentWriteOutMedium segmentWriteOutMedium; @@ -50,14 +65,20 @@ public static LongColumnSerializer create( private final ByteOrder byteOrder; private final CompressionStrategy compression; private final CompressionFactory.LongEncodingStrategy encoding; + private final BitmapSerdeFactory bitmapSerdeFactory; + private ColumnarLongsSerializer writer; + private ByteBufferWriter nullValueBitmapWriter; + private MutableBitmap nullRowsBitmap; + private int rowCount = 0; private LongColumnSerializer( SegmentWriteOutMedium segmentWriteOutMedium, String filenameBase, ByteOrder byteOrder, CompressionStrategy compression, - CompressionFactory.LongEncodingStrategy encoding + CompressionFactory.LongEncodingStrategy encoding, + BitmapSerdeFactory bitmapSerdeFactory ) { this.segmentWriteOutMedium = segmentWriteOutMedium; @@ -65,6 +86,7 @@ private LongColumnSerializer( this.byteOrder = byteOrder; this.compression = compression; this.encoding = encoding; + this.bitmapSerdeFactory = bitmapSerdeFactory; } @Override @@ -78,24 +100,43 @@ public void open() throws IOException compression ); writer.open(); + nullValueBitmapWriter = new ByteBufferWriter<>( + segmentWriteOutMedium, + bitmapSerdeFactory.getObjectStrategy() + ); + nullValueBitmapWriter.open(); + nullRowsBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); } @Override - public void serialize(Object obj) throws IOException + public void serialize(@Nullable Object obj) throws IOException { - long val = (obj == null) ? 0 : ((Number) obj).longValue(); - writer.add(val); + if (obj == null) { + nullRowsBitmap.add(rowCount); + writer.add(0L); + } else { + writer.add(((Number) obj).longValue()); + } + rowCount++; } @Override public long getSerializedSize() throws IOException { - return writer.getSerializedSize(); + nullValueBitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullRowsBitmap)); + long bitmapSize = nullRowsBitmap.isEmpty() + ? 0L + : nullValueBitmapWriter.getSerializedSize(); + return Integer.BYTES + writer.getSerializedSize() + bitmapSize; } @Override public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException { + channel.write(ByteBuffer.wrap(Ints.toByteArray((int) writer.getSerializedSize()))); writer.writeTo(channel, smoosher); + if (!nullRowsBitmap.isEmpty()) { + nullValueBitmapWriter.writeTo(channel, smoosher); + } } } diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index 8aeeca25395d..0d3ed447f08a 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -21,6 +21,7 @@ import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.java.util.common.guava.Comparators; import io.druid.query.dimension.DimensionSpec; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.Indexed; @@ -28,10 +29,13 @@ import io.druid.segment.incremental.TimeAndDimsHolder; import javax.annotation.Nullable; +import java.util.Comparator; import java.util.List; +import java.util.Objects; public class LongDimensionIndexer implements DimensionIndexer { + public static final Comparator LONG_COMPARATOR = Comparators.naturalNullsFirst(); @Override public Long processRowValsToUnsortedEncodedKeyComponent(Object dimValues) @@ -97,13 +101,20 @@ public long getLong() { final Object[] dims = currEntry.get().getDims(); - if (dimIndex >= dims.length) { - return 0L; + if (dimIndex >= dims.length || dims[dimIndex] == null) { + return 0; } return (Long) dims[dimIndex]; } + @Override + public boolean isNull() + { + final Object[] dims = currEntry.get().getDims(); + return dimIndex >= dims.length || dims[dimIndex] == null; + } + @SuppressWarnings("deprecation") @Nullable @Override @@ -131,19 +142,19 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) @Override public int compareUnsortedEncodedKeyComponents(@Nullable Long lhs, @Nullable Long rhs) { - return DimensionHandlerUtils.nullToZero(lhs).compareTo(DimensionHandlerUtils.nullToZero(rhs)); + return LONG_COMPARATOR.compare(lhs, rhs); } @Override public boolean checkUnsortedEncodedKeyComponentsEqual(@Nullable Long lhs, @Nullable Long rhs) { - return DimensionHandlerUtils.nullToZero(lhs).equals(DimensionHandlerUtils.nullToZero(rhs)); + return Objects.equals(lhs, rhs); } @Override public int getUnsortedEncodedKeyComponentHashCode(@Nullable Long key) { - return DimensionHandlerUtils.nullToZero(key).hashCode(); + return DimensionHandlerUtils.nullToZeroLong(key).hashCode(); } @Override diff --git a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java index ce515ad1020b..a3a6d94ea4fb 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionMergerV9.java @@ -23,8 +23,8 @@ import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; import io.druid.segment.data.CompressionFactory; +import io.druid.segment.serde.LongGenericColumnPartSerdeV2; import io.druid.segment.data.CompressionStrategy; -import io.druid.segment.serde.LongGenericColumnPartSerde; import io.druid.segment.writeout.SegmentWriteOutMedium; import java.io.IOException; @@ -33,6 +33,7 @@ public class LongDimensionMergerV9 implements DimensionMergerV9 { + protected String dimensionName; protected final IndexSpec indexSpec; protected LongColumnSerializer serializer; @@ -58,7 +59,13 @@ protected void setupEncodedValueWriter(SegmentWriteOutMedium segmentWriteOutMedi { final CompressionStrategy metCompression = indexSpec.getMetricCompression(); final CompressionFactory.LongEncodingStrategy longEncoding = indexSpec.getLongEncoding(); - this.serializer = LongColumnSerializer.create(segmentWriteOutMedium, dimensionName, metCompression, longEncoding); + this.serializer = LongColumnSerializer.create( + segmentWriteOutMedium, + dimensionName, + metCompression, + longEncoding, + indexSpec.getBitmapSerdeFactory() + ); serializer.open(); } @@ -89,7 +96,6 @@ public void writeIndexes(List segmentRowNumConversions) throws IOExce @Override public boolean canSkip() { - // a long column can never be all null return false; } @@ -99,10 +105,11 @@ public ColumnDescriptor makeColumnDescriptor() throws IOException final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); builder.setValueType(ValueType.LONG); builder.addSerde( - LongGenericColumnPartSerde.serializerBuilder() - .withByteOrder(IndexIO.BYTE_ORDER) - .withDelegate(serializer) - .build() + LongGenericColumnPartSerdeV2.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withBitmapSerdeFactory(indexSpec.getBitmapSerdeFactory()) + .withDelegate(serializer) + .build() ); return builder.build(); } diff --git a/processing/src/main/java/io/druid/segment/NilColumnValueSelector.java b/processing/src/main/java/io/druid/segment/NilColumnValueSelector.java index 26ba95715743..c19208711a42 100644 --- a/processing/src/main/java/io/druid/segment/NilColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/NilColumnValueSelector.java @@ -85,6 +85,12 @@ public Class classOfObject() return Object.class; } + @Override + public boolean isNull() + { + return true; + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index 7c8969305444..56ad73fcae1e 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -20,7 +20,7 @@ package io.druid.segment; import com.google.common.base.Predicate; -import com.google.common.base.Strings; +import io.druid.common.config.NullHandling; import io.druid.query.filter.ValueMatcher; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.data.IndexedInts; @@ -103,7 +103,7 @@ public IdLookup idLookup() @Override public int lookupId(String name) { - return Strings.isNullOrEmpty(name) ? 0 : -1; + return NullHandling.isNullOrEquivalent(name) ? 0 : -1; } @Nullable diff --git a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java index 2cf5a6c2173a..17e5da9822d7 100644 --- a/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java @@ -23,7 +23,7 @@ * This interface is convenient for implementation of "object-sourcing" {@link ColumnValueSelector}s, it provides * default implementations for all {@link ColumnValueSelector}'s methods except {@link #getObject()} and {@link * #classOfObject()}. - * + *

* This interface should appear ONLY in "implements" clause or anonymous class creation, but NOT in "user" code, where * {@link BaseObjectColumnValueSelector} must be used instead. */ @@ -76,4 +76,17 @@ default long getLong() } return ((Number) value).longValue(); } + + /** + * @deprecated This method is marked as deprecated in ObjectColumnSelector to minimize the probability of accidential + * calling. "Polymorphism" of ObjectColumnSelector should be used only when operating on {@link ColumnValueSelector} + * objects. + */ + @Deprecated + @Override + default boolean isNull() + { + T value = getObject(); + return value == null; + } } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index 0f6dd5182f43..5b0e533e8a00 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -37,10 +37,15 @@ public interface QueryableIndex extends ColumnSelector, Closeable { Interval getDataInterval(); + int getNumRows(); + Indexed getAvailableDimensions(); + BitmapFactory getBitmapFactoryForDimensions(); + Metadata getMetadata(); + Map getDimensionHandlers(); /** diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 675077021632..431c050bde8a 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -39,8 +39,8 @@ import io.druid.segment.column.FloatsColumn; import io.druid.segment.column.LongsColumn; import io.druid.segment.column.ValueType; -import io.druid.segment.data.ImmutableBitmapValues; import io.druid.segment.data.BitmapValues; +import io.druid.segment.data.ImmutableBitmapValues; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedIterable; import io.druid.segment.data.ListIndexed; @@ -270,11 +270,14 @@ public Rowboat next() Object[] metricArray = new Object[numMetrics]; for (int i = 0; i < metricArray.length; ++i) { if (metrics[i] instanceof FloatsColumn) { - metricArray[i] = ((GenericColumn) metrics[i]).getFloatSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) metrics[i]; + metricArray[i] = genericColumn.isNull(currRow) ? null : genericColumn.getFloatSingleValueRow(currRow); } else if (metrics[i] instanceof DoublesColumn) { - metricArray[i] = ((GenericColumn) metrics[i]).getDoubleSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) metrics[i]; + metricArray[i] = genericColumn.isNull(currRow) ? null : genericColumn.getDoubleSingleValueRow(currRow); } else if (metrics[i] instanceof LongsColumn) { - metricArray[i] = ((GenericColumn) metrics[i]).getLongSingleValueRow(currRow); + GenericColumn genericColumn = (GenericColumn) metrics[i]; + metricArray[i] = genericColumn.isNull(currRow) ? null : genericColumn.getLongSingleValueRow(currRow); } else if (metrics[i] instanceof ComplexColumn) { metricArray[i] = ((ComplexColumn) metrics[i]).getRowValue(currRow); } diff --git a/processing/src/main/java/io/druid/segment/Segment.java b/processing/src/main/java/io/druid/segment/Segment.java index 1da21f047e35..42cff575193f 100644 --- a/processing/src/main/java/io/druid/segment/Segment.java +++ b/processing/src/main/java/io/druid/segment/Segment.java @@ -30,8 +30,11 @@ public interface Segment extends Closeable { String getIdentifier(); + Interval getDataInterval(); + QueryableIndex asQueryableIndex(); + StorageAdapter asStorageAdapter(); /** diff --git a/processing/src/main/java/io/druid/segment/StorageAdapter.java b/processing/src/main/java/io/druid/segment/StorageAdapter.java index f6658082b490..887aaafa06ff 100644 --- a/processing/src/main/java/io/druid/segment/StorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/StorageAdapter.java @@ -34,8 +34,11 @@ public interface StorageAdapter extends CursorFactory { @PublicApi String getSegmentIdentifier(); + Interval getInterval(); + Indexed getAvailableDimensions(); + Iterable getAvailableMetrics(); /** @@ -47,12 +50,15 @@ public interface StorageAdapter extends CursorFactory * @return */ int getDimensionCardinality(String column); + DateTime getMinTime(); + DateTime getMaxTime(); @Nullable Comparable getMinValue(String column); @Nullable Comparable getMaxValue(String column); + Capabilities getCapabilities(); /** @@ -73,7 +79,10 @@ public interface StorageAdapter extends CursorFactory * @return type name */ String getColumnTypeName(String column); + int getNumRows(); + DateTime getMaxIngestedEventTime(); + Metadata getMetadata(); } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java index cd4db8a3db46..fe3387f5548b 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -77,6 +77,20 @@ public int compareSortedEncodedKeyComponents(int[] lhs, int[] rhs) return retVal; } + private boolean isNUllRow(int[] row, Indexed encodings) + { + if (row == null) { + return true; + } + for (int i : row) { + if (encodings.get(i) != null) { + // Non-Null value + return false; + } + } + return true; + } + @Override public void validateSortedEncodedKeyComponents( int[] lhs, @@ -86,7 +100,7 @@ public void validateSortedEncodedKeyComponents( ) throws SegmentValidationException { if (lhs == null || rhs == null) { - if (lhs != null || rhs != null) { + if (!isNUllRow(lhs, lhsEncodings) || !isNUllRow(rhs, rhsEncodings)) { throw new SegmentValidationException( "Expected nulls, found %s and %s", Arrays.toString(lhs), diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index bbe89d603388..306a9e2c0e92 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -21,11 +21,11 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.MutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -58,13 +58,18 @@ public class StringDimensionIndexer implements DimensionIndexer { - private static final Function STRING_TRANSFORMER = o -> o != null ? o.toString() : null; + private static final Function EMPTY_TO_NULL_IF_NEEDED = o -> o != null + ? NullHandling.emptyToNullIfNeeded(o.toString()) + : null; + + private static final int ABSENT_VALUE_ID = -1; private static final int[] EMPTY_INT_ARRAY = new int[]{}; private static class DimensionDictionary { private String minValue = null; private String maxValue = null; + private int idForNull = ABSENT_VALUE_ID; private final Object2IntMap valueToId = new Object2IntOpenHashMap<>(); @@ -80,37 +85,59 @@ public DimensionDictionary() public int getId(String value) { synchronized (lock) { - return valueToId.getInt(Strings.nullToEmpty(value)); + if (value == null) { + return idForNull; + } + return valueToId.getInt(value); } } public String getValue(int id) { synchronized (lock) { - return Strings.emptyToNull(idToValue.get(id)); + if (id == idForNull) { + return null; + } + return idToValue.get(id); + } + } + + public boolean contains(String value) + { + synchronized (lock) { + if (value == null) { + return idForNull != ABSENT_VALUE_ID; + } + return valueToId.containsKey(value); } } public int size() { synchronized (lock) { - return valueToId.size(); + return idToValue.size(); } } public int add(String originalValue) { - String value = Strings.nullToEmpty(originalValue); synchronized (lock) { - int prev = valueToId.getInt(value); + if (originalValue == null) { + if (idForNull == ABSENT_VALUE_ID) { + idForNull = size(); + idToValue.add(null); + } + return idForNull; + } + int prev = valueToId.getInt(originalValue); if (prev >= 0) { return prev; } final int index = size(); - valueToId.put(value, index); - idToValue.add(value); - minValue = minValue == null || minValue.compareTo(value) > 0 ? value : minValue; - maxValue = maxValue == null || maxValue.compareTo(value) < 0 ? value : maxValue; + valueToId.put(originalValue, index); + idToValue.add(originalValue); + minValue = minValue == null || minValue.compareTo(originalValue) > 0 ? originalValue : minValue; + maxValue = maxValue == null || maxValue.compareTo(originalValue) < 0 ? originalValue : maxValue; return index; } } @@ -145,15 +172,18 @@ private static class SortedDimensionDictionary public SortedDimensionDictionary(List idToValue, int length) { - Object2IntSortedMap sortedMap = new Object2IntRBTreeMap<>(); + Object2IntSortedMap sortedMap = new Object2IntRBTreeMap<>(Comparators.naturalNullsFirst()); for (int id = 0; id < length; id++) { - sortedMap.put(idToValue.get(id), id); + String value = idToValue.get(id); + sortedMap.put(value, id); } - this.sortedVals = Lists.newArrayList(sortedMap.keySet()); + + this.sortedVals = new ArrayList<>(length); this.idToIndex = new int[length]; this.indexToId = new int[length]; int index = 0; - for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext();) { + sortedVals.addAll(sortedMap.keySet()); + for (IntIterator iterator = sortedMap.values().iterator(); iterator.hasNext(); ) { int id = iterator.nextInt(); idToIndex[id] = index; indexToId[index] = id; @@ -173,7 +203,7 @@ public int getSortedIdFromUnsortedId(int id) public String getValueFromSortedId(int index) { - return Strings.emptyToNull(sortedVals.get(index)); + return sortedVals.get(index); } } @@ -194,19 +224,22 @@ public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues) final int oldDictSize = dimLookup.size(); if (dimValues == null) { - dimLookup.add(null); - encodedDimensionValues = null; + if (!dimLookup.contains(null)) { + encodedDimensionValues = new int[]{dimLookup.add(null)}; + } else { + encodedDimensionValues = new int[]{dimLookup.getId(null)}; + } } else if (dimValues instanceof List) { List dimValuesList = (List) dimValues; if (dimValuesList.isEmpty()) { dimLookup.add(null); encodedDimensionValues = EMPTY_INT_ARRAY; } else if (dimValuesList.size() == 1) { - encodedDimensionValues = new int[]{dimLookup.add(STRING_TRANSFORMER.apply(dimValuesList.get(0)))}; + encodedDimensionValues = new int[]{dimLookup.add(EMPTY_TO_NULL_IF_NEEDED.apply(dimValuesList.get(0)))}; } else { final String[] dimensionValues = new String[dimValuesList.size()]; for (int i = 0; i < dimValuesList.size(); i++) { - dimensionValues[i] = STRING_TRANSFORMER.apply(dimValuesList.get(i)); + dimensionValues[i] = EMPTY_TO_NULL_IF_NEEDED.apply(dimValuesList.get(i)); } if (multiValueHandling.needSorting()) { // Sort multival row by their unencoded values first. @@ -231,7 +264,7 @@ public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues) encodedDimensionValues = pos == retVal.length ? retVal : Arrays.copyOf(retVal, pos); } } else { - encodedDimensionValues = new int[]{dimLookup.add(STRING_TRANSFORMER.apply(dimValues))}; + encodedDimensionValues = new int[]{dimLookup.add(EMPTY_TO_NULL_IF_NEEDED.apply(dimValues))}; } // If dictionary size has changed, the sorted lookup is no longer valid. @@ -280,7 +313,7 @@ public String get(int index) public int indexOf(String value) { int id = getEncodedValue(value, false); - return id < 0 ? -1 : getSortedEncodedValueFromUnsorted(id); + return id < 0 ? ABSENT_VALUE_ID : getSortedEncodedValueFromUnsorted(id); } @Override @@ -389,7 +422,7 @@ public IndexedInts getRow() final int nullId = getEncodedValue(null, false); if (nullId > -1) { if (nullIdIntArray == null) { - nullIdIntArray = new int[] {nullId}; + nullIdIntArray = new int[]{nullId}; } row = nullIdIntArray; rowSize = 1; @@ -583,22 +616,18 @@ public Object convertUnsortedEncodedKeyComponentToActualArrayOrList(int[] key, b return null; } if (key.length == 1) { - String val = getActualValue(key[0], false); - val = Strings.nullToEmpty(val); - return val; + return getActualValue(key[0], false); } else { if (asList) { List rowVals = new ArrayList<>(key.length); for (int id : key) { - String val = getActualValue(id, false); - rowVals.add(Strings.nullToEmpty(val)); + rowVals.add(getActualValue(id, false)); } return rowVals; } else { String[] rowArray = new String[key.length]; for (int i = 0; i < key.length; i++) { - String val = getActualValue(key[i], false); - rowArray[i] = Strings.nullToEmpty(val); + rowArray[i] = getActualValue(key[i], false); } return rowArray; } diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 8e97666bc184..3e767bbb1edd 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -20,7 +20,6 @@ package io.druid.segment; import com.google.common.base.Splitter; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -28,6 +27,7 @@ import io.druid.collections.spatial.ImmutableRTree; import io.druid.collections.spatial.RTree; import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; @@ -63,8 +63,11 @@ public class StringDimensionMergerV9 implements DimensionMergerV9 { private static final Logger log = new Logger(StringDimensionMergerV9.class); - private static final Indexed EMPTY_STR_DIM_VAL = new ArrayIndexed<>(new String[]{""}, String.class); - private static final int[] EMPTY_STR_DIM_ARRAY = new int[]{0}; + protected static final Indexed NULL_STR_DIM_VAL = new ArrayIndexed<>( + new String[]{(String) null}, + String.class + ); + protected static final int[] NULL_STR_DIM_ARRAY = new int[]{0}; private static final Splitter SPLITTER = Splitter.on(","); private ColumnarIntsSerializer encodedValueWriter; @@ -137,7 +140,7 @@ public void writeMergedValueMetadata(List adapters) throws IOE convertMissingValues = dimHasValues && dimAbsentFromSomeIndex; /* - * Ensure the empty str is always in the dictionary if the dimension was missing from one index but + * Ensure the null str is always in the dictionary if the dimension was missing from one index but * has non-null values in another index. * This is done so that MMappedIndexRowIterable can convert null columns to empty strings * later on, to allow rows from indexes without a particular dimension to merge correctly with @@ -145,7 +148,7 @@ public void writeMergedValueMetadata(List adapters) throws IOE */ if (convertMissingValues && !hasNull) { hasNull = true; - dimValueLookups[adapters.size()] = dimValueLookup = EMPTY_STR_DIM_VAL; + dimValueLookups[adapters.size()] = dimValueLookup = NULL_STR_DIM_VAL; numMergeIndex++; } @@ -184,7 +187,7 @@ private void writeDictionary(Iterable dictionaryValues) throws IOExcepti { for (String value : dictionaryValues) { dictionaryWriter.write(value); - value = Strings.emptyToNull(value); + value = NullHandling.emptyToNullIfNeeded(value); if (dictionarySize == 0) { firstDictionaryValue = value; } @@ -231,7 +234,7 @@ public int[] convertSegmentRowValuesToMergedRowValues(int[] segmentRow, int segm // For strings, convert missing values to null/empty if conversion flag is set // But if bitmap/dictionary is not used, always convert missing to 0 if (dimVals == null) { - return convertMissingValues ? EMPTY_STR_DIM_ARRAY : null; + return convertMissingValues ? NULL_STR_DIM_ARRAY : null; } int[] newDimVals = new int[dimVals.length]; @@ -376,6 +379,7 @@ void mergeBitmaps( bitmapWriter.write(bmpFactory.makeImmutableBitmap(mergedIndexes)); if (hasSpatial) { + String dimVal = dictionaryWriter.get(dictId); if (dimVal != null) { List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); diff --git a/processing/src/main/java/io/druid/segment/column/Column.java b/processing/src/main/java/io/druid/segment/column/Column.java index f74ddd2e072e..a025b4988871 100644 --- a/processing/src/main/java/io/druid/segment/column/Column.java +++ b/processing/src/main/java/io/druid/segment/column/Column.java @@ -37,10 +37,14 @@ static boolean storeDoubleAsFloat() ColumnCapabilities getCapabilities(); int getLength(); + DictionaryEncodedColumn getDictionaryEncoding(); GenericColumn getGenericColumn(); + ComplexColumn getComplexColumn(); + BitmapIndex getBitmapIndex(); + SpatialIndex getSpatialIndex(); } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java index 8cab3a11aa9d..7eccf0d90bb9 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java @@ -26,9 +26,13 @@ public interface ColumnCapabilities ValueType getType(); boolean isDictionaryEncoded(); + boolean isRunLengthEncoded(); + boolean hasBitmapIndexes(); + boolean hasSpatialIndexes(); + boolean hasMultipleValues(); } diff --git a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 0793c40f249c..7f100f02d388 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -113,7 +113,6 @@ public static class Builder { private ValueType valueType = null; private Boolean hasMultipleValues = null; - private final List parts = Lists.newArrayList(); public Builder setValueType(ValueType valueType) @@ -145,7 +144,11 @@ public Builder addSerde(ColumnPartSerde serde) public ColumnDescriptor build() { Preconditions.checkNotNull(valueType, "must specify a valueType"); - return new ColumnDescriptor(valueType, hasMultipleValues == null ? false : hasMultipleValues, parts); + return new ColumnDescriptor( + valueType, + hasMultipleValues == null ? false : hasMultipleValues, + parts + ); } } } diff --git a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java index 0f5d718bf1c3..cd9974e249a4 100644 --- a/processing/src/main/java/io/druid/segment/column/ComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java @@ -31,7 +31,9 @@ public interface ComplexColumn extends BaseColumn { Class getClazz(); + String getTypeName(); + Object getRowValue(int rowNum); @Override diff --git a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index 51b54e07cdd5..7b19d47482b7 100644 --- a/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -32,11 +32,17 @@ public interface DictionaryEncodedColumn extends BaseColumn { int length(); + boolean hasMultipleValues(); + int getSingleValueRow(int rowNum); + IndexedInts getMultiValueRow(int rowNum); + @Nullable ActualType lookupName(int id); + int lookupId(ActualType name); + int getCardinality(); DimensionSelector makeDimensionSelector(ReadableOffset offset, @Nullable ExtractionFn extractionFn); diff --git a/processing/src/main/java/io/druid/segment/column/DoublesColumn.java b/processing/src/main/java/io/druid/segment/column/DoublesColumn.java index 5601d39f4c93..55c499698ab9 100644 --- a/processing/src/main/java/io/druid/segment/column/DoublesColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DoublesColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.data.ColumnarDoubles; @@ -28,10 +29,12 @@ public class DoublesColumn implements GenericColumn { private final ColumnarDoubles column; + private final ImmutableBitmap nullValueBitmap; - public DoublesColumn(ColumnarDoubles columnarDoubles) + public DoublesColumn(ColumnarDoubles columnarDoubles, ImmutableBitmap nullValueBitmap) { column = columnarDoubles; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -43,7 +46,7 @@ public int length() @Override public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { - return column.makeColumnValueSelector(offset); + return column.makeColumnValueSelector(offset, nullValueBitmap); } @Override @@ -64,6 +67,12 @@ public double getDoubleSingleValueRow(int rowNum) return column.get(rowNum); } + @Override + public boolean isNull(int rowNum) + { + return nullValueBitmap.get(rowNum); + } + @Override public void close() { diff --git a/processing/src/main/java/io/druid/segment/column/FloatsColumn.java b/processing/src/main/java/io/druid/segment/column/FloatsColumn.java index 5bc381c19c17..51a869b1c98d 100644 --- a/processing/src/main/java/io/druid/segment/column/FloatsColumn.java +++ b/processing/src/main/java/io/druid/segment/column/FloatsColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.data.ColumnarFloats; @@ -29,10 +30,15 @@ public class FloatsColumn implements GenericColumn { private final ColumnarFloats column; + private final ImmutableBitmap nullValueBitmap; - public FloatsColumn(final ColumnarFloats column) + public FloatsColumn( + final ColumnarFloats column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -44,7 +50,7 @@ public int length() @Override public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { - return column.makeColumnValueSelector(offset); + return column.makeColumnValueSelector(offset, nullValueBitmap); } @Override @@ -65,6 +71,12 @@ public double getDoubleSingleValueRow(int rowNum) return (double) column.get(rowNum); } + @Override + public boolean isNull(int rowNum) + { + return nullValueBitmap.get(rowNum); + } + @Override public void close() { diff --git a/processing/src/main/java/io/druid/segment/column/GenericColumn.java b/processing/src/main/java/io/druid/segment/column/GenericColumn.java index 2c277db7a0d2..c00a3054e9ad 100644 --- a/processing/src/main/java/io/druid/segment/column/GenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/GenericColumn.java @@ -37,6 +37,9 @@ public interface GenericColumn extends BaseColumn, HotLoopCallee @CalledFromHotLoop double getDoubleSingleValueRow(int rowNum); + @CalledFromHotLoop + boolean isNull(int rowNum); + @Override void close(); } diff --git a/processing/src/main/java/io/druid/segment/column/LongsColumn.java b/processing/src/main/java/io/druid/segment/column/LongsColumn.java index 4f67a4b9e565..2b5c5b864713 100644 --- a/processing/src/main/java/io/druid/segment/column/LongsColumn.java +++ b/processing/src/main/java/io/druid/segment/column/LongsColumn.java @@ -19,6 +19,7 @@ package io.druid.segment.column; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.data.ColumnarLongs; @@ -29,10 +30,15 @@ public class LongsColumn implements GenericColumn { private final ColumnarLongs column; + private final ImmutableBitmap nullValueBitmap; - public LongsColumn(final ColumnarLongs column) + public LongsColumn( + final ColumnarLongs column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override @@ -44,7 +50,7 @@ public int length() @Override public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { - return column.makeColumnValueSelector(offset); + return column.makeColumnValueSelector(offset, nullValueBitmap); } @Override @@ -65,6 +71,12 @@ public double getDoubleSingleValueRow(int rowNum) return (double) column.get(rowNum); } + @Override + public boolean isNull(int rowNum) + { + return nullValueBitmap.get(rowNum); + } + @Override public void close() { diff --git a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index 3d67beb22a2b..51a337878d34 100644 --- a/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -21,7 +21,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; -import com.google.common.base.Strings; import io.druid.java.util.common.guava.CloseQuietly; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.ValueMatcher; @@ -43,7 +42,7 @@ import java.util.BitSet; /** -*/ + */ public class SimpleDictionaryEncodedColumn implements DictionaryEncodedColumn { private final ColumnarInts column; @@ -86,10 +85,10 @@ public IndexedInts getMultiValueRow(int rowNum) } @Override + @Nullable public String lookupName(int id) { - //Empty to Null will ensure that null and empty are equivalent for extraction function - return Strings.emptyToNull(cachedLookups.get(id)); + return cachedLookups.get(id); } @Override @@ -177,6 +176,13 @@ public ValueMatcher makeValueMatcher(Predicate predicate) return DimensionSelectorUtils.makeValueMatcherGeneric(this, predicate); } + @Override + public boolean isNull() + { + IndexedInts row = getRow(); + return row == null || row.size() == 0; + } + @Nullable @Override public Object getObject() @@ -282,6 +288,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) }; } + @Override + public boolean isNull() + { + return getObject() == null; + } + @Override public Object getObject() { diff --git a/processing/src/main/java/io/druid/segment/data/ColumnarDoubles.java b/processing/src/main/java/io/druid/segment/data/ColumnarDoubles.java index 407ef6947407..1deff8d5e094 100644 --- a/processing/src/main/java/io/druid/segment/data/ColumnarDoubles.java +++ b/processing/src/main/java/io/druid/segment/data/ColumnarDoubles.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.DoubleColumnSelector; @@ -33,12 +34,13 @@ public interface ColumnarDoubles extends Closeable { int size(); + double get(int index); @Override void close(); - default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalDoubleColumnSelector implements DoubleColumnSelector, HistoricalColumnSelector { @@ -54,6 +56,12 @@ public double getDouble(int offset) return ColumnarDoubles.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/data/ColumnarFloats.java b/processing/src/main/java/io/druid/segment/data/ColumnarFloats.java index 582b254ca0d1..4b06272f3f6e 100644 --- a/processing/src/main/java/io/druid/segment/data/ColumnarFloats.java +++ b/processing/src/main/java/io/druid/segment/data/ColumnarFloats.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.FloatColumnSelector; @@ -33,13 +34,15 @@ public interface ColumnarFloats extends Closeable { int size(); + float get(int index); + void fill(int index, float[] toFill); @Override void close(); - default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalFloatColumnSelector implements FloatColumnSelector, HistoricalColumnSelector { @@ -49,6 +52,12 @@ public float getFloat() return ColumnarFloats.this.get(offset.getOffset()); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public double getDouble(int offset) { diff --git a/processing/src/main/java/io/druid/segment/data/ColumnarLongs.java b/processing/src/main/java/io/druid/segment/data/ColumnarLongs.java index 5a60fc50744b..f581a833004a 100644 --- a/processing/src/main/java/io/druid/segment/data/ColumnarLongs.java +++ b/processing/src/main/java/io/druid/segment/data/ColumnarLongs.java @@ -19,6 +19,7 @@ package io.druid.segment.data; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.ColumnValueSelector; import io.druid.segment.LongColumnSelector; @@ -33,13 +34,15 @@ public interface ColumnarLongs extends Closeable { int size(); + long get(int index); + void fill(int index, long[] toFill); @Override void close(); - default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + default ColumnValueSelector makeColumnValueSelector(ReadableOffset offset, ImmutableBitmap nullValueBitmap) { class HistoricalLongColumnSelector implements LongColumnSelector, HistoricalColumnSelector { @@ -55,6 +58,12 @@ public double getDouble(int offset) return ColumnarLongs.this.get(offset); } + @Override + public boolean isNull() + { + return nullValueBitmap.get(offset.getOffset()); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 2bbfa9ef6568..41416e7c9799 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -19,9 +19,9 @@ package io.druid.segment.data; -import com.google.common.base.Strings; import com.google.common.primitives.Ints; import io.druid.collections.ResourceHolder; +import io.druid.common.config.NullHandling; import io.druid.common.utils.SerializerUtils; import io.druid.io.Channels; import io.druid.java.util.common.IAE; @@ -31,12 +31,12 @@ import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.io.smoosh.FileSmoosher; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; -import io.druid.segment.writeout.HeapByteBufferWriteOutBytes; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.segment.serde.MetaSerdeHelper; import io.druid.segment.serde.Serializer; -import it.unimi.dsi.fastutil.bytes.ByteArrays; +import io.druid.segment.writeout.HeapByteBufferWriteOutBytes; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; @@ -98,18 +98,20 @@ public Class getClazz() } @Override + @Nullable public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) { - return StringUtils.fromUtf8(buffer, numBytes); + if (numBytes < 0) { + // nulBytes will be -1 for null values. + return null; + } + return NullHandling.emptyToNullIfNeeded(StringUtils.fromUtf8Nullable(buffer, numBytes)); } @Override public byte[] toBytes(String val) { - if (Strings.isNullOrEmpty(val)) { - return ByteArrays.EMPTY_ARRAY; - } - return StringUtils.toUtf8(val); + return StringUtils.toUtf8Nullable(NullHandling.nullToEmptyIfNeeded(val)); } @Override @@ -309,8 +311,6 @@ private int indexOf(Indexed indexed, T value) throw new UnsupportedOperationException("Reverse lookup not allowed."); } - value = (value != null && value.equals("")) ? null : value; - int minIndex = 0; int maxIndex = size - 1; while (minIndex <= maxIndex) { @@ -370,11 +370,8 @@ public GenericIndexed.BufferIndexed singleThreaded() private T copyBufferAndGet(ByteBuffer valueBuffer, int startOffset, int endOffset) { - final int size = endOffset - startOffset; - if (size == 0) { - return null; - } ByteBuffer copyValueBuffer = valueBuffer.asReadOnlyBuffer(); + final int size = endOffset > startOffset ? endOffset - startOffset : copyValueBuffer.get(startOffset - Ints.BYTES); copyValueBuffer.position(startOffset); // fromByteBuffer must not modify the buffer limit return strategy.fromByteBuffer(copyValueBuffer, size); @@ -413,11 +410,11 @@ public int size() T bufferedIndexedGet(ByteBuffer copyValueBuffer, int startOffset, int endOffset) { - final int size = endOffset - startOffset; + final int size = endOffset > startOffset + ? endOffset - startOffset + : copyValueBuffer.get(startOffset - Ints.BYTES); lastReadSize = size; - if (size == 0) { - return null; - } + // ObjectStrategy.fromByteBuffer() is allowed to reset the limit of the buffer. So if the limit is changed, // position() call in the next line could throw an exception, if the position is set beyond the new limit. clear() // sets the limit to the maximum possible, the capacity. It is safe to reset the limit to capacity, because the @@ -496,8 +493,7 @@ private static GenericIndexed fromIterableVersionOne( allowReverseLookup = false; } - // for compatibility with the format, but this field is unused - valuesOut.writeInt(0); + valuesOut.writeInt(next == null ? -1 : 0); strategy.writeTo(next, valuesOut); headerOut.writeInt(Ints.checkedCast(valuesOut.size())); @@ -561,7 +557,7 @@ public T get(final int index) final int endOffset; if (index == 0) { - startOffset = 4; + startOffset = Ints.BYTES; endOffset = headerBuffer.getInt(0); } else { int headerPosition = (index - 1) * Ints.BYTES; diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index 31822e13cc51..18c602260ba4 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -214,9 +214,8 @@ public void write(T objectToWrite) throws IOException } ++numWritten; - // for compatibility with the format (see GenericIndexed javadoc for description of the format), but this field is - // unused. - valuesOut.writeInt(0); + + valuesOut.writeInt(objectToWrite == null ? -1 : 0); strategy.writeTo(objectToWrite, valuesOut); if (!requireMultipleFiles) { diff --git a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java index fb4135e2ab23..2dd391e18cce 100644 --- a/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java @@ -22,6 +22,7 @@ import io.druid.guice.annotations.ExtensionPoint; import io.druid.segment.writeout.WriteOutBytes; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Comparator; @@ -33,19 +34,23 @@ public interface ObjectStrategy extends Comparator /** * Convert values from their underlying byte representation. - * + *

* Implementations of this method may change the given buffer's mark, or limit, and position. - * + *

* Implementations of this method may not store the given buffer in a field of the "deserialized" object, * need to use {@link ByteBuffer#slice()}, {@link ByteBuffer#asReadOnlyBuffer()} or {@link ByteBuffer#duplicate()} in * this case. * - * @param buffer buffer to read value from + * @param buffer buffer to read value from * @param numBytes number of bytes used to store the value, starting at buffer.position() + * * @return an object created from the given byte buffer representation */ + @Nullable T fromByteBuffer(ByteBuffer buffer, int numBytes); - byte[] toBytes(T val); + + @Nullable + byte[] toBytes(@Nullable T val); /** * Reads 4-bytes numBytes from the given buffer, and then delegates to {@link #fromByteBuffer(ByteBuffer, int)}. @@ -62,6 +67,9 @@ default T fromByteBufferWithSize(ByteBuffer buffer) default void writeTo(T val, WriteOutBytes out) throws IOException { - out.write(toBytes(val)); + byte[] bytes = toBytes(val); + if (bytes != null) { + out.write(toBytes(val)); + } } } diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index cb76d50189d7..4125927f8937 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -79,6 +79,9 @@ public Class getClazz() @Override public ImmutableBitmap fromByteBuffer(ByteBuffer buffer, int numBytes) { + if (numBytes == 0) { + return null; + } buffer.limit(buffer.position() + numBytes); return new WrappedImmutableRoaringBitmap(new ImmutableRoaringBitmap(buffer)); } diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index c9fc4262a1ee..c8115b804a39 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.Pair; import io.druid.query.BitmapResultFactory; import io.druid.query.extraction.ExtractionFn; @@ -148,7 +149,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasLowerBound()) { startIndex = 0; } else { - final int found = bitmapIndex.getIndex(boundDimFilter.getLower()); + final int found = bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getLower())); if (found >= 0) { startIndex = boundDimFilter.isLowerStrict() ? found + 1 : found; } else { @@ -159,7 +160,7 @@ private static Pair getStartEndIndexes( if (!boundDimFilter.hasUpperBound()) { endIndex = bitmapIndex.getCardinality(); } else { - final int found = bitmapIndex.getIndex(boundDimFilter.getUpper()); + final int found = bitmapIndex.getIndex(NullHandling.emptyToNullIfNeeded(boundDimFilter.getUpper())); if (found >= 0) { endIndex = boundDimFilter.isUpperStrict() ? found : found + 1; } else { @@ -249,9 +250,10 @@ private boolean doesMatch(String input) { if (input == null) { return (!boundDimFilter.hasLowerBound() - || (boundDimFilter.getLower().isEmpty() && !boundDimFilter.isLowerStrict())) // lower bound allows null + || (NullHandling.isNullOrEquivalent(boundDimFilter.getLower()) && !boundDimFilter.isLowerStrict())) + // lower bound allows null && (!boundDimFilter.hasUpperBound() - || !boundDimFilter.getUpper().isEmpty() + || !NullHandling.isNullOrEquivalent(boundDimFilter.getUpper()) || !boundDimFilter.isUpperStrict()); // upper bound allows null } int lowerComparing = 1; diff --git a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java index 4acea0f29376..c13d9edd3aa0 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExpressionFilter.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import io.druid.common.config.NullHandling; import io.druid.math.expr.Evals; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -108,7 +109,8 @@ public T getBitmapResult(final BitmapIndexSelector selector, final BitmapRes value -> expr.eval(identifierName -> { // There's only one binding, and it must be the single column, so it can safely be ignored in production. assert column.equals(identifierName); - return value; + // convert null to Empty before passing to expressions if needed. + return NullHandling.nullToEmptyIfNeeded(value); }).asBoolean() ); } diff --git a/processing/src/main/java/io/druid/segment/filter/InFilter.java b/processing/src/main/java/io/druid/segment/filter/InFilter.java index 7e737616dadd..869e94e37ef7 100644 --- a/processing/src/main/java/io/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/InFilter.java @@ -20,7 +20,6 @@ package io.druid.segment.filter; import com.google.common.base.Predicate; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.BitmapResultFactory; @@ -165,9 +164,9 @@ private DruidPredicateFactory getPredicateFactory() public Predicate makeStringPredicate() { if (extractionFn != null) { - return input -> values.contains(Strings.nullToEmpty(extractionFn.apply(input))); + return input -> values.contains(extractionFn.apply(input)); } else { - return input -> values.contains(Strings.nullToEmpty(input)); + return input -> values.contains(input); } } diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index a5984f31b5c8..99b953c6270f 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -19,9 +19,9 @@ package io.druid.segment.filter; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.query.BitmapResultFactory; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; @@ -90,7 +90,10 @@ private Iterable getBitmapIterable(final BitmapIndexSelector se { if (isSimpleEquals()) { // Verify that dimension equals prefix. - return ImmutableList.of(selector.getBitmapIndex(dimension, likeMatcher.getPrefix())); + return ImmutableList.of(selector.getBitmapIndex( + dimension, + NullHandling.emptyToNullIfNeeded(likeMatcher.getPrefix()) + )); } else if (isSimplePrefix()) { // Verify that dimension startsWith prefix, and is accepted by likeMatcher.matchesSuffixOnly. final BitmapIndex bitmapIndex = selector.getBitmapIndex(dimension); @@ -140,16 +143,24 @@ private IntIterable getDimValueIndexIterableForPrefixMatch( final Indexed dimValues ) { - final String lower = Strings.nullToEmpty(likeMatcher.getPrefix()); - final String upper = Strings.nullToEmpty(likeMatcher.getPrefix()) + Character.MAX_VALUE; + + final String lower = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()); + final String upper = NullHandling.nullToEmptyIfNeeded(likeMatcher.getPrefix()) + Character.MAX_VALUE; + final int startIndex; // inclusive final int endIndex; // exclusive - final int lowerFound = bitmapIndex.getIndex(lower); - startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1); + if (lower == null) { + // For Null values + startIndex = bitmapIndex.getIndex(null); + endIndex = startIndex + 1; + } else { + final int lowerFound = bitmapIndex.getIndex(lower); + startIndex = lowerFound >= 0 ? lowerFound : -(lowerFound + 1); - final int upperFound = bitmapIndex.getIndex(upper); - endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1); + final int upperFound = bitmapIndex.getIndex(upper); + endIndex = upperFound >= 0 ? upperFound + 1 : -(upperFound + 1); + } return new IntIterable() { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 09cb0cfd3dee..833e84136d24 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -151,22 +151,28 @@ public ColumnValueSelector makeColumnValueSelector(final String column) final ComplexMetricExtractor extractor = serde.getExtractor(); return new ColumnValueSelector() { + @Override + public boolean isNull() + { + return in.get().getMetric(column) == null; + } + @Override public long getLong() { - return in.get().getMetric(column).longValue(); + return DimensionHandlerUtils.nullToZeroLong(in.get().getMetric(column)).longValue(); } @Override public float getFloat() { - return in.get().getMetric(column).floatValue(); + return DimensionHandlerUtils.nullToZeroFloat(in.get().getMetric(column)).floatValue(); } @Override public double getDouble() { - return in.get().getMetric(column).doubleValue(); + return DimensionHandlerUtils.nullToZeroDouble(in.get().getMetric(column)).doubleValue(); } @Override @@ -461,6 +467,9 @@ protected abstract Integer addToFacts( protected abstract double getMetricDoubleValue(int rowOffset, int aggOffset); + protected abstract boolean isNull(int rowOffset, int aggOffset); + + @Override public void close() { @@ -552,9 +561,10 @@ TimeAndDims toTimeAndDims(InputRow row) DimensionHandler handler = DimensionHandlerUtils.getHandlerFromCapabilities(dimension, capabilities, null); desc = addNewDimension(dimension, capabilities, handler); } + Object raw = row.getRaw(dimension); DimensionHandler handler = desc.getHandler(); DimensionIndexer indexer = desc.getIndexer(); - Object dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent(row.getRaw(dimension)); + Object dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent(raw); // Set column capabilities as data is coming in if (!capabilities.hasMultipleValues() && dimsKey != null && handler.getLengthOfEncodedKeyComponent(dimsKey) > 1) { @@ -1397,6 +1407,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("index", IncrementalIndex.this); } + + @Override + public boolean isNull() + { + return IncrementalIndex.this.isNull(currEntry.getValue(), metricIndex); + } } private class ObjectMetricColumnSelector implements ObjectColumnSelector @@ -1458,6 +1474,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("index", IncrementalIndex.this); } + + @Override + public boolean isNull() + { + return IncrementalIndex.this.isNull(currEntry.getValue(), metricIndex); + } } private class DoubleMetricColumnSelector implements DoubleColumnSelector @@ -1477,6 +1499,12 @@ public double getDouble() return getMetricDoubleValue(currEntry.getValue(), metricIndex); } + @Override + public boolean isNull() + { + return IncrementalIndex.this.isNull(currEntry.getValue(), metricIndex); + } + @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java index d965b80aa120..23afec83166a 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java @@ -121,6 +121,12 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { // nothing to inspect } + + @Override + public boolean isNull() + { + return false; + } } return new TimeLongColumnSelector(); } diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index 3d45604b69c9..d377634e0bc8 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -313,6 +313,15 @@ public double getMetricDoubleValue(int rowOffset, int aggOffset) return agg.getDouble(bb, indexAndOffset[1] + aggOffsetInBuffer[aggOffset]); } + @Override + public boolean isNull(int rowOffset, int aggOffset) + { + BufferAggregator agg = getAggs()[aggOffset]; + int[] indexAndOffset = indexAndOffsets.get(rowOffset); + ByteBuffer bb = aggBuffers.get(indexAndOffset[0]).get(); + return agg.isNull(bb, indexAndOffset[1] + aggOffsetInBuffer[aggOffset]); + } + /** * NOTE: This is NOT thread-safe with add... so make sure all the adding is DONE before closing */ diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index c41b63a3376b..1a377ea826be 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -286,6 +286,12 @@ protected double getMetricDoubleValue(int rowOffset, int aggOffset) return concurrentGet(rowOffset)[aggOffset].getDouble(); } + @Override + public boolean isNull(int rowOffset, int aggOffset) + { + return concurrentGet(rowOffset)[aggOffset].isNull(); + } + /** * Clear out maps to allow GC * NOTE: This is NOT thread-safe with add... so make sure all the adding is DONE before closing diff --git a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java index 4c7d7b1fd86c..6662e60269bc 100644 --- a/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java @@ -34,7 +34,10 @@ @JsonSubTypes.Type(name = "float", value = FloatGenericColumnPartSerde.class), @JsonSubTypes.Type(name = "long", value = LongGenericColumnPartSerde.class), @JsonSubTypes.Type(name = "double", value = DoubleGenericColumnPartSerde.class), - @JsonSubTypes.Type(name = "stringDictionary", value = DictionaryEncodedColumnPartSerde.class) + @JsonSubTypes.Type(name = "stringDictionary", value = DictionaryEncodedColumnPartSerde.class), + @JsonSubTypes.Type(name = "floatV2", value = FloatGenericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "longV2", value = LongGenericColumnPartSerdeV2.class), + @JsonSubTypes.Type(name = "doubleV2", value = DoubleGenericColumnPartSerdeV2.class), }) public interface ColumnPartSerde { diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java index b251456834a2..b8e3609aa146 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java @@ -28,5 +28,6 @@ public interface ComplexMetricExtractor { Class extractedClass(); + Object extractValue(InputRow inputRow, String metricName); } diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java index 3a545ec675e4..f4b7c7c3eb28 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerde.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Supplier; import io.druid.segment.DoubleColumnSerializer; +import io.druid.segment.IndexIO; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ValueType; @@ -105,7 +106,8 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); builder.setType(ValueType.DOUBLE) .setHasMultipleValues(false) - .setGenericColumn(new DoubleGenericColumnSupplier(column)); + .setGenericColumn(new DoubleGenericColumnSupplier(column, IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap())); } }; diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java new file mode 100644 index 000000000000..d768eae1a15e --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeV2.java @@ -0,0 +1,163 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.DoubleColumnSerializer; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ColumnarDoubles; +import io.druid.segment.data.CompressedColumnarDoublesSuppliers; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; + +public class DoubleGenericColumnPartSerdeV2 implements ColumnPartSerde +{ + private final ByteOrder byteOrder; + private Serializer serialize; + private final BitmapSerdeFactory bitmapSerdeFactory; + + @JsonCreator + public static DoubleGenericColumnPartSerdeV2 getDoubleGenericColumnPartSerde( + @JsonProperty("byteOrder") ByteOrder byteOrder, + @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + ) + { + return new DoubleGenericColumnPartSerdeV2(byteOrder, + bitmapSerdeFactory != null + ? bitmapSerdeFactory + : new BitmapSerde.LegacyBitmapSerdeFactory(), null + ); + } + + @JsonProperty + public ByteOrder getByteOrder() + { + return byteOrder; + } + + @JsonProperty + public BitmapSerdeFactory getBitmapSerdeFactory() + { + return bitmapSerdeFactory; + } + + public DoubleGenericColumnPartSerdeV2( + ByteOrder byteOrder, + BitmapSerdeFactory bitmapSerdeFactory, + Serializer serialize + ) + { + this.byteOrder = byteOrder; + this.bitmapSerdeFactory = bitmapSerdeFactory; + this.serialize = serialize; + } + + @Override + public Serializer getSerializer() + { + return serialize; + } + + @Override + public Deserializer getDeserializer() + { + return (buffer, builder, columnConfig) -> { + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final Supplier column = CompressedColumnarDoublesSuppliers.fromByteBuffer( + buffer, + byteOrder + ); + + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); + } else { + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); + } + builder.setType(ValueType.DOUBLE) + .setHasMultipleValues(false) + .setGenericColumn(new DoubleGenericColumnSupplier(column, bitmap)); + }; + } + + public static SerializerBuilder serializerBuilder() + { + return new SerializerBuilder(); + } + + public static class SerializerBuilder + { + private ByteOrder byteOrder = null; + private DoubleColumnSerializer delegate = null; + private BitmapSerdeFactory bitmapSerdeFactory = null; + + public SerializerBuilder withByteOrder(final ByteOrder byteOrder) + { + this.byteOrder = byteOrder; + return this; + } + + public SerializerBuilder withDelegate(final DoubleColumnSerializer delegate) + { + this.delegate = delegate; + return this; + } + + public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public DoubleGenericColumnPartSerdeV2 build() + { + return new DoubleGenericColumnPartSerdeV2( + byteOrder, + bitmapSerdeFactory, + new Serializer() + { + @Override + public long getSerializedSize() throws IOException + { + return delegate.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + { + delegate.writeTo(channel, fileSmoosher); + } + } + ); + } + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java index 148bbcbbd609..ae4ea7a50ba5 100644 --- a/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/DoubleGenericColumnSupplier.java @@ -20,6 +20,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.DoublesColumn; import io.druid.segment.data.ColumnarDoubles; @@ -28,15 +29,20 @@ public class DoubleGenericColumnSupplier implements Supplier { private final Supplier column; + private final ImmutableBitmap nullValueBitmap; - public DoubleGenericColumnSupplier(Supplier column) + public DoubleGenericColumnSupplier( + Supplier column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override public GenericColumn get() { - return new DoublesColumn(column.get()); + return new DoublesColumn(column.get(), nullValueBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java index c62c5e744623..0615b1cb3e82 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.segment.FloatColumnSerializer; +import io.druid.segment.IndexIO; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ValueType; @@ -105,7 +106,11 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); builder.setType(ValueType.FLOAT) .setHasMultipleValues(false) - .setGenericColumn(new FloatGenericColumnSupplier(column)); + .setGenericColumn(new FloatGenericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )); } }; } diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java new file mode 100644 index 000000000000..bae4747d3ff8 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerdeV2.java @@ -0,0 +1,162 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.FloatColumnSerializer; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.CompressedColumnarFloatsSupplier; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; + +/** + */ +public class FloatGenericColumnPartSerdeV2 implements ColumnPartSerde +{ + @JsonCreator + public static FloatGenericColumnPartSerdeV2 createDeserializer( + @JsonProperty("byteOrder") ByteOrder byteOrder, + @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + ) + { + return new FloatGenericColumnPartSerdeV2( + byteOrder, + bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), + null + ); + } + + private final ByteOrder byteOrder; + private final BitmapSerdeFactory bitmapSerdeFactory; + private Serializer serializer; + + private FloatGenericColumnPartSerdeV2( + ByteOrder byteOrder, + BitmapSerdeFactory bitmapSerdeFactory, + Serializer serializer + ) + { + this.byteOrder = byteOrder; + this.bitmapSerdeFactory = bitmapSerdeFactory; + this.serializer = serializer; + } + + @JsonProperty + public ByteOrder getByteOrder() + { + return byteOrder; + } + + @JsonProperty + public BitmapSerdeFactory getBitmapSerdeFactory() + { + return bitmapSerdeFactory; + } + + public static SerializerBuilder serializerBuilder() + { + return new SerializerBuilder(); + } + + public static class SerializerBuilder + { + private ByteOrder byteOrder = null; + private FloatColumnSerializer delegate = null; + private BitmapSerdeFactory bitmapSerdeFactory = null; + + public SerializerBuilder withByteOrder(final ByteOrder byteOrder) + { + this.byteOrder = byteOrder; + return this; + } + + public SerializerBuilder withDelegate(final FloatColumnSerializer delegate) + { + this.delegate = delegate; + return this; + } + + public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public FloatGenericColumnPartSerdeV2 build() + { + return new FloatGenericColumnPartSerdeV2( + byteOrder, bitmapSerdeFactory, + new Serializer() + { + @Override + public long getSerializedSize() throws IOException + { + return delegate.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher fileSmoosher) throws IOException + { + delegate.writeTo(channel, fileSmoosher); + } + } + ); + } + + } + + @Override + public Serializer getSerializer() + { + return serializer; + } + + @Override + public Deserializer getDeserializer() + { + return (buffer, builder, columnConfig) -> { + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedColumnarFloatsSupplier column = CompressedColumnarFloatsSupplier.fromByteBuffer( + buffer, + byteOrder + ); + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); + } else { + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); + } + builder.setType(ValueType.FLOAT) + .setHasMultipleValues(false) + .setGenericColumn(new FloatGenericColumnSupplier(column, bitmap)); + + }; + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java index 98568c8d2bd7..0d40a02cadb4 100644 --- a/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java @@ -20,6 +20,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.FloatsColumn; import io.druid.segment.data.CompressedColumnarFloatsSupplier; @@ -29,15 +30,20 @@ public class FloatGenericColumnSupplier implements Supplier { private final CompressedColumnarFloatsSupplier column; + private final ImmutableBitmap nullValueBitmap; - public FloatGenericColumnSupplier(CompressedColumnarFloatsSupplier column) + public FloatGenericColumnSupplier( + CompressedColumnarFloatsSupplier column, + ImmutableBitmap nullValueBitmap + ) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override public GenericColumn get() { - return new FloatsColumn(column.get()); + return new FloatsColumn(column.get(), nullValueBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java index a73fde430e13..ea86ada53a42 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.segment.IndexIO; import io.druid.segment.LongColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ColumnConfig; @@ -105,7 +106,11 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo ); builder.setType(ValueType.LONG) .setHasMultipleValues(false) - .setGenericColumn(new LongGenericColumnSupplier(column)); + .setGenericColumn(new LongGenericColumnSupplier( + column, + IndexIO.LEGACY_FACTORY.getBitmapFactory() + .makeEmptyImmutableBitmap() + )); } }; } diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java new file mode 100644 index 000000000000..dfd37dde4afb --- /dev/null +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerdeV2.java @@ -0,0 +1,160 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.java.util.common.io.smoosh.FileSmoosher; +import io.druid.segment.LongColumnSerializer; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerde; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.CompressedColumnarLongsSupplier; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteOrder; +import java.nio.channels.WritableByteChannel; + +/** + */ +public class LongGenericColumnPartSerdeV2 implements ColumnPartSerde +{ + @JsonCreator + public static LongGenericColumnPartSerdeV2 createDeserializer( + @JsonProperty("byteOrder") ByteOrder byteOrder, + @Nullable @JsonProperty("bitmapSerdeFactory") BitmapSerdeFactory bitmapSerdeFactory + ) + { + return new LongGenericColumnPartSerdeV2( + byteOrder, + bitmapSerdeFactory != null ? bitmapSerdeFactory : new BitmapSerde.LegacyBitmapSerdeFactory(), + null + ); + } + + private final ByteOrder byteOrder; + private final BitmapSerdeFactory bitmapSerdeFactory; + private Serializer serializer; + + private LongGenericColumnPartSerdeV2( + ByteOrder byteOrder, + BitmapSerdeFactory bitmapSerdeFactory, Serializer serializer + ) + { + this.byteOrder = byteOrder; + this.bitmapSerdeFactory = bitmapSerdeFactory; + this.serializer = serializer; + } + + @JsonProperty + public ByteOrder getByteOrder() + { + return byteOrder; + } + + @JsonProperty + public BitmapSerdeFactory getBitmapSerdeFactory() + { + return bitmapSerdeFactory; + } + + public static SerializerBuilder serializerBuilder() + { + return new SerializerBuilder(); + } + + public static class SerializerBuilder + { + private ByteOrder byteOrder = null; + private LongColumnSerializer delegate = null; + private BitmapSerdeFactory bitmapSerdeFactory = null; + + public SerializerBuilder withByteOrder(final ByteOrder byteOrder) + { + this.byteOrder = byteOrder; + return this; + } + + public SerializerBuilder withDelegate(final LongColumnSerializer delegate) + { + this.delegate = delegate; + return this; + } + + public SerializerBuilder withBitmapSerdeFactory(BitmapSerdeFactory bitmapSerdeFactory) + { + this.bitmapSerdeFactory = bitmapSerdeFactory; + return this; + } + + public LongGenericColumnPartSerdeV2 build() + { + return new LongGenericColumnPartSerdeV2( + byteOrder, bitmapSerdeFactory, + new Serializer() + { + @Override + public long getSerializedSize() throws IOException + { + return delegate.getSerializedSize(); + } + + @Override + public void writeTo(WritableByteChannel channel, FileSmoosher smoosher) throws IOException + { + delegate.writeTo(channel, smoosher); + } + } + ); + } + } + + @Override + public Serializer getSerializer() + { + return serializer; + } + + @Override + public Deserializer getDeserializer() + { + return (buffer, builder, columnConfig) -> { + int offset = buffer.getInt(); + int initialPos = buffer.position(); + final CompressedColumnarLongsSupplier column = CompressedColumnarLongsSupplier.fromByteBuffer( + buffer, + byteOrder + ); + buffer.position(initialPos + offset); + final ImmutableBitmap bitmap; + if (buffer.hasRemaining()) { + bitmap = bitmapSerdeFactory.getObjectStrategy().fromByteBufferWithSize(buffer); + } else { + bitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyImmutableBitmap(); + } + builder.setType(ValueType.LONG) + .setHasMultipleValues(false) + .setGenericColumn(new LongGenericColumnSupplier(column, bitmap)); + + }; + } +} diff --git a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java index c5861bb5b477..e71657b2a46a 100644 --- a/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java @@ -20,6 +20,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.LongsColumn; import io.druid.segment.data.CompressedColumnarLongsSupplier; @@ -29,15 +30,17 @@ public class LongGenericColumnSupplier implements Supplier { private final CompressedColumnarLongsSupplier column; + private final ImmutableBitmap nullValueBitmap; - public LongGenericColumnSupplier(CompressedColumnarLongsSupplier column) + public LongGenericColumnSupplier(CompressedColumnarLongsSupplier column, ImmutableBitmap nullValueBitmap) { this.column = column; + this.nullValueBitmap = nullValueBitmap; } @Override public GenericColumn get() { - return new LongsColumn(column.get()); + return new LongsColumn(column.get(), nullValueBitmap); } } diff --git a/processing/src/main/java/io/druid/segment/transform/Transform.java b/processing/src/main/java/io/druid/segment/transform/Transform.java index fe3b043d5197..bb8f46f832d3 100644 --- a/processing/src/main/java/io/druid/segment/transform/Transform.java +++ b/processing/src/main/java/io/druid/segment/transform/Transform.java @@ -26,10 +26,10 @@ * A row transform that is part of a {@link TransformSpec}. Transforms allow adding new fields to input rows. Each * one has a "name" (the name of the new field) which can be referred to by DimensionSpecs, AggregatorFactories, etc. * Each also has a "row function", which produces values for this new field based on looking at the entire input row. - * + *

* If a transform has the same name as a field in an input row, then it will shadow the original field. Transforms * that shadow fields may still refer to the fields they shadow. This can be used to transform a field "in-place". - * + *

* Transforms do have some limitations. They can only refer to fields present in the actual input rows; in particular, * they cannot refer to other transforms. And they cannot remove fields, only add them. However, they can shadow a * field with another field containing all nulls, which will act similarly to removing the field. diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java index 7cbb277721a9..0fd23ebffcdc 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionColumnValueSelector.java @@ -75,4 +75,10 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) inspector.visit("expression", expression); inspector.visit("bindings", bindings); } + + @Override + public boolean isNull() + { + return getObject() == null; + } } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java new file mode 100644 index 000000000000..6c132273ee81 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectSelector.java @@ -0,0 +1,168 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; +import io.druid.math.expr.Expr; +import io.druid.math.expr.ExprEval; +import io.druid.math.expr.Parser; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import io.druid.segment.AggregatorNullHandling; +import io.druid.segment.BaseObjectColumnValueSelector; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ColumnValueSelector; +import io.druid.segment.DimensionSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedInts; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Map; + +public class ExpressionObjectSelector implements ObjectColumnSelector +{ + private final Expr expression; + private final Expr.ObjectBinding bindings; + + private ExpressionObjectSelector(Expr.ObjectBinding bindings, Expr expression) + { + this.bindings = Preconditions.checkNotNull(bindings, "bindings"); + this.expression = Preconditions.checkNotNull(expression, "expression"); + } + + public static ExpressionObjectSelector from(ColumnSelectorFactory columnSelectorFactory, Expr expression) + { + return new ExpressionObjectSelector(createBindings(columnSelectorFactory, expression), expression); + } + + private static Expr.ObjectBinding createBindings(ColumnSelectorFactory columnSelectorFactory, Expr expression) + { + final Map> suppliers = Maps.newHashMap(); + for (String columnName : Parser.findRequiredBindings(expression)) { + final ColumnCapabilities columnCapabilities = columnSelectorFactory.getColumnCapabilities(columnName); + final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; + final Supplier supplier; + if (nativeType == ValueType.FLOAT) { + final ColumnValueSelector columnValueSelector = columnSelectorFactory.makeColumnValueSelector(columnName); + supplier = AggregatorNullHandling.getNullableSupplier(columnValueSelector::getFloat, columnValueSelector); + } else if (nativeType == ValueType.LONG) { + final ColumnValueSelector columnValueSelector = columnSelectorFactory.makeColumnValueSelector(columnName); + supplier = AggregatorNullHandling.getNullableSupplier(columnValueSelector::getLong, columnValueSelector); + } else if (nativeType == ValueType.DOUBLE) { + final ColumnValueSelector columnValueSelector = columnSelectorFactory.makeColumnValueSelector(columnName); + supplier = AggregatorNullHandling.getNullableSupplier(columnValueSelector::getDouble, columnValueSelector); + } else if (nativeType == ValueType.STRING) { + supplier = supplierFromDimensionSelector( + columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName)) + ); + } else if (nativeType == null) { + // Unknown ValueType. Try making an Object selector and see if that gives us anything useful. + supplier = supplierFromObjectSelector(columnSelectorFactory.makeColumnValueSelector(columnName)); + } else { + // Unhandleable ValueType (COMPLEX). + supplier = null; + } + + if (supplier != null) { + suppliers.put(columnName, supplier); + } + } + + return Parser.withSuppliers(suppliers); + } + + @VisibleForTesting + @Nonnull + static Supplier supplierFromDimensionSelector(final DimensionSelector selector) + { + Preconditions.checkNotNull(selector, "selector"); + return () -> { + final IndexedInts row = selector.getRow(); + if (row.size() == 0) { + // Treat empty multi-value rows as nulls. + return NullHandling.defaultStringValue(); + } else if (row.size() == 1) { + return NullHandling.nullToEmptyIfNeeded(selector.lookupName(row.get(0))); + } else { + // Can't handle multi-value rows in expressions. + // Treat them as nulls until we think of something better to do. + return NullHandling.defaultStringValue(); + } + }; + } + + @VisibleForTesting + @Nullable + static Supplier supplierFromObjectSelector(final BaseObjectColumnValueSelector selector) + { + if (selector == null) { + // Missing column. + return Suppliers.ofInstance(NullHandling.defaultStringValue()); + } + + final Class clazz = selector.classOfObject(); + if (Number.class.isAssignableFrom(clazz) || String.class.isAssignableFrom(clazz)) { + // Number, String supported as-is. + return selector::getObject; + } else if (clazz.isAssignableFrom(Number.class) || clazz.isAssignableFrom(String.class)) { + // Might be Numbers and Strings. Use a selector that double-checks. + return () -> { + final Object val = selector.getObject(); + if (val instanceof String) { + return NullHandling.nullToEmptyIfNeeded((String) val); + } else if (val instanceof Number) { + return val; + } else { + return NullHandling.defaultStringValue(); + } + }; + } else { + // No numbers or strings. + return null; + } + } + + @Override + public Class classOfObject() + { + return ExprEval.class; + } + + @Override + public ExprEval getObject() + { + return expression.eval(bindings); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("expression", expression); + inspector.visit("bindings", bindings); + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 2c2a3a80a93f..5f8652643aa5 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -21,10 +21,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; @@ -77,6 +77,12 @@ public double getDouble() return baseSelector.getDouble(); } + @Override + public boolean isNull() + { + return baseSelector.getObject().isNull(); + } + @Override public float getFloat() { @@ -198,7 +204,7 @@ class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelecto @Override protected String getValue() { - return Strings.emptyToNull(baseSelector.getObject().asString()); + return NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString()); } @Override @@ -214,7 +220,7 @@ class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSele @Override protected String getValue() { - return extractionFn.apply(Strings.emptyToNull(baseSelector.getObject().asString())); + return extractionFn.apply(NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString())); } @Override diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java index 873b4a331470..f2051ae9f64b 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleLongInputCachingExpressionColumnValueSelector.java @@ -143,4 +143,10 @@ private ExprEval eval(final long value) bindings.set(value); return expression.eval(bindings); } + + @Override + public boolean isNull() + { + return getObject().isNull(); + } } diff --git a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java index fba53f7d3d10..71557231cfe8 100644 --- a/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/SingleStringInputCachingExpressionColumnValueSelector.java @@ -133,6 +133,12 @@ private ExprEval eval() return expression.eval(bindings); } + @Override + public boolean isNull() + { + return eval().isNull(); + } + public static class LruEvalCache { private final Expr expression; diff --git a/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java new file mode 100644 index 000000000000..f510db561ce7 --- /dev/null +++ b/processing/src/test/java/io/druid/guice/NullHandlingHelperInjectionTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.guice; + +import com.google.inject.Injector; +import io.druid.common.config.NullHandling; +import org.junit.Assert; +import org.junit.Test; + +public class NullHandlingHelperInjectionTest +{ + public static String NULL_HANDLING_CONFIG_STRING = ("druid.generic.useDefaultValueForNull"); + + @Test + public void testNullHandlingHelperUseDefaultValues() + { + String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); + try { + System.setProperty(NULL_HANDLING_CONFIG_STRING, "true"); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(true, NullHandling.useDefaultValuesForNull()); + } + finally { + if (prev != null) { + resetNullHandlingHelper(prev); + } + } + } + + @Test + public void testNullHandlingHelperNoDefaultValues() + { + String prev = System.getProperty(NULL_HANDLING_CONFIG_STRING); + try { + System.setProperty(NULL_HANDLING_CONFIG_STRING, "false"); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(false, NullHandling.useDefaultValuesForNull()); + } + finally { + if (prev != null) { + resetNullHandlingHelper(prev); + } + } + } + + private void resetNullHandlingHelper(String prev) + { + System.setProperty(NULL_HANDLING_CONFIG_STRING, prev); + Injector injector = GuiceInjectors.makeStartupInjector(); + Assert.assertEquals(Boolean.valueOf(prev), NullHandling.useDefaultValuesForNull()); + } + +} diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java index 9a96d28db49c..85d7cd110565 100644 --- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.google.common.util.concurrent.MoreExecutors; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.MapInputRowParser; @@ -48,6 +49,7 @@ import io.druid.segment.IndexBuilder; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.TestHelper; import io.druid.segment.incremental.IncrementalIndexSchema; import org.junit.After; import org.junit.Assert; @@ -284,10 +286,22 @@ public void testNumericEvolutionTimeseriesAggregation() ); // Only nonexistent(4) - Assert.assertEquals( - timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), - runQuery(query, factory, ImmutableList.of(index4)) - ); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), + runQuery(query, factory, ImmutableList.of(index4)) + ); + } else { + Map result = Maps.newHashMap(); + result.put("a", null); + result.put("b", null); + result.put("c", null); + result.put("d", null); + Assert.assertEquals( + timeseriesResult(result), + runQuery(query, factory, ImmutableList.of(index4)) + ); + } // string(1) + long(2) + float(3) + nonexistent(4) // Note: Expressions implicitly cast strings to numbers, leading to the a/b vs c/d difference. @@ -354,7 +368,14 @@ public void testNumericEvolutionFiltering() // Only nonexistent(4) Assert.assertEquals( - timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L)), + timeseriesResult(TestHelper.createExpectedMap( + "a", + NullHandling.useDefaultValuesForNull() ? 0L : null, + "b", + NullHandling.useDefaultValuesForNull() ? 0.0 : null, + "c", + 0L + )), runQuery(query, factory, ImmutableList.of(index4)) ); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java index bcf85922ea24..3d7c5a6a393d 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMaxAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testDoubleMaxAggregator() { - DoubleMaxAggregator agg = (DoubleMaxAggregator) doubleMaxAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleMaxAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -72,9 +72,9 @@ public void testDoubleMaxAggregator() @Test public void testDoubleMaxBufferAggregator() { - DoubleMaxBufferAggregator agg = (DoubleMaxBufferAggregator) doubleMaxAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = doubleMaxAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -106,13 +106,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMaxAggregator agg) + private void aggregate(TestDoubleColumnSelectorImpl selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMaxBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestDoubleColumnSelectorImpl selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java index d198989e9807..f5337d3363f0 100644 --- a/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleMinAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testDoubleMinAggregator() { - DoubleMinAggregator agg = (DoubleMinAggregator) doubleMinAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleMinAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -72,9 +72,9 @@ public void testDoubleMinAggregator() @Test public void testDoubleMinBufferAggregator() { - DoubleMinBufferAggregator agg = (DoubleMinBufferAggregator) doubleMinAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = doubleMinAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Doubles.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -106,13 +106,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMinAggregator agg) + private void aggregate(TestDoubleColumnSelectorImpl selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestDoubleColumnSelectorImpl selector, DoubleMinBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestDoubleColumnSelectorImpl selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 584f08a2b810..26914a4c318c 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.js.JavaScriptConfig; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; @@ -224,9 +225,9 @@ public ColumnCapabilities getColumnCapabilities(String columnName) private void assertValues(FilteredAggregator agg, TestFloatColumnSelector selector, double... expectedVals) { - Assert.assertEquals(0.0d, agg.get()); - Assert.assertEquals(0.0d, agg.get()); - Assert.assertEquals(0.0d, agg.get()); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0.0d : null, agg.get()); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0.0d : null, agg.get()); for (double expectedVal : expectedVals) { aggregate(selector, agg); Assert.assertEquals(expectedVal, agg.get()); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java index 90a8f27f8b92..e025a4c126a7 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMaxAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testLongMaxAggregator() { - LongMaxAggregator agg = (LongMaxAggregator) longMaxAggFactory.factorize(colSelectorFactory); + Aggregator agg = longMaxAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -72,9 +72,9 @@ public void testLongMaxAggregator() @Test public void testLongMaxBufferAggregator() { - LongMaxBufferAggregator agg = (LongMaxBufferAggregator) longMaxAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = longMaxAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -106,13 +106,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestLongColumnSelector selector, LongMaxAggregator agg) + private void aggregate(TestLongColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestLongColumnSelector selector, LongMaxBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestLongColumnSelector selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java index b907c08019b9..f8ed6b1b09ed 100644 --- a/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongMinAggregationTest.java @@ -57,7 +57,7 @@ public void setup() @Test public void testLongMinAggregator() { - LongMinAggregator agg = (LongMinAggregator) longMinAggFactory.factorize(colSelectorFactory); + Aggregator agg = longMinAggFactory.factorize(colSelectorFactory); aggregate(selector, agg); aggregate(selector, agg); @@ -72,9 +72,9 @@ public void testLongMinAggregator() @Test public void testLongMinBufferAggregator() { - LongMinBufferAggregator agg = (LongMinBufferAggregator) longMinAggFactory.factorizeBuffered(colSelectorFactory); + BufferAggregator agg = longMinAggFactory.factorizeBuffered(colSelectorFactory); - ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES]); + ByteBuffer buffer = ByteBuffer.wrap(new byte[Longs.BYTES + Byte.BYTES]); agg.init(buffer, 0); aggregate(selector, agg, buffer, 0); @@ -106,13 +106,13 @@ public void testEqualsAndHashCode() throws Exception Assert.assertFalse(one.equals(two)); } - private void aggregate(TestLongColumnSelector selector, LongMinAggregator agg) + private void aggregate(TestLongColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); } - private void aggregate(TestLongColumnSelector selector, LongMinBufferAggregator agg, ByteBuffer buff, int position) + private void aggregate(TestLongColumnSelector selector, BufferAggregator agg, ByteBuffer buff, int position) { agg.aggregate(buff, position); selector.increment(); diff --git a/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java b/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java index 967086f155f8..8721a5ad7780 100644 --- a/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/MetricManipulatorFnsTest.java @@ -50,6 +50,12 @@ public long getLong() { return longVal; } + + @Override + public boolean isNull() + { + return false; + } } ); LongMinAggregatorFactory longMinAggregatorFactory = new LongMinAggregatorFactory(NAME, FIELD); @@ -89,6 +95,12 @@ public long getLong() { return longVal; } + + @Override + public boolean isNull() + { + return false; + } } ); constructorArrays.add( diff --git a/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java b/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java index 9e7e1dd93215..4f2cac74f8cb 100644 --- a/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestDoubleColumnSelectorImpl.java @@ -39,6 +39,12 @@ public double getDouble() return doubles[index]; } + @Override + public boolean isNull() + { + return false; + } + public void increment() { ++index; diff --git a/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java b/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java index 8c0aa532d8f5..dc1c68caeb03 100644 --- a/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java @@ -38,6 +38,12 @@ public float getFloat() return floats[index]; } + @Override + public boolean isNull() + { + return false; + } + public void increment() { ++index; diff --git a/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java b/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java index 2a78bb500521..aca323a66c8b 100644 --- a/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestLongColumnSelector.java @@ -38,6 +38,12 @@ public long getLong() return longs[index]; } + @Override + public boolean isNull() + { + return false; + } + public void increment() { ++index; diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index e7ddc3f94d1b..09aa51b267a6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.js.JavaScriptConfig; import io.druid.query.ColumnSelectorPlus; @@ -429,8 +430,8 @@ public void testAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { aggregate(selectorList, agg); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); - Assert.assertEquals(7L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get())); } @Test @@ -473,8 +474,8 @@ public void testBufferAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { bufferAggregate(selectorList, agg, buf, pos); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); - Assert.assertEquals(7L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7L : 6L, rowAggregatorFactoryRounded.finalizeComputation(agg.get(buf, pos))); } @Test @@ -553,11 +554,11 @@ public void testCombineValues() aggregate(selector2, agg2); } - Assert.assertEquals(4.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 4.0 : 3.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); Assert.assertEquals( - 7.0, + NullHandling.useDefaultValuesForNull() ? 7.0 : 6.0, (Double) rowAggregatorFactory.finalizeComputation( rowAggregatorFactory.combine( agg1.get(), diff --git a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 8aba7090aba7..15d419e06d19 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestDoubleColumnSelectorImpl; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleFirstAggregator() { - DoubleFirstAggregator agg = (DoubleFirstAggregator) doubleFirstAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { - DoubleFirstBufferAggregator agg = (DoubleFirstBufferAggregator) doubleFirstAggFactory.factorizeBuffered( + BufferAggregator agg = doubleFirstAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleFirstAggFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleFirstCombiningAggregator() { - DoubleFirstAggregator agg = (DoubleFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { - DoubleFirstBufferAggregator agg = (DoubleFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleFirstAggFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - DoubleFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - DoubleFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java index ff0f00f5eb0b..719c16980dd6 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestFloatColumnSelector; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleFirstAggregator() { - FloatFirstAggregator agg = (FloatFirstAggregator) floatFirstAggregatorFactory.factorize(colSelectorFactory); + Aggregator agg = floatFirstAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { - FloatFirstBufferAggregator agg = (FloatFirstBufferAggregator) floatFirstAggregatorFactory.factorizeBuffered( + BufferAggregator agg = floatFirstAggregatorFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleFirstCombiningAggregator() { - FloatFirstAggregator agg = (FloatFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { - FloatFirstBufferAggregator agg = (FloatFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - FloatFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - FloatFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java index 36a8b5cb3f6a..6c48dbfd5ac1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -70,7 +72,7 @@ public void setup() @Test public void testLongFirstAggregator() { - LongFirstAggregator agg = (LongFirstAggregator) longFirstAggFactory.factorize(colSelectorFactory); + Aggregator agg = longFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -88,7 +90,7 @@ public void testLongFirstAggregator() @Test public void testLongFirstBufferAggregator() { - LongFirstBufferAggregator agg = (LongFirstBufferAggregator) longFirstAggFactory.factorizeBuffered( + BufferAggregator agg = longFirstAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longFirstAggFactory.getMaxIntermediateSize()]); @@ -118,7 +120,7 @@ public void testCombine() @Test public void testLongFirstCombiningAggregator() { - LongFirstAggregator agg = (LongFirstAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -137,7 +139,7 @@ public void testLongFirstCombiningAggregator() @Test public void testLongFirstCombiningBufferAggregator() { - LongFirstBufferAggregator agg = (LongFirstBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longFirstAggFactory.getMaxIntermediateSize()]); @@ -167,7 +169,7 @@ public void testSerde() throws Exception } private void aggregate( - LongFirstAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -177,7 +179,7 @@ private void aggregate( } private void aggregate( - LongFirstBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java index c5453e18171a..423472df77b1 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestDoubleColumnSelectorImpl; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleLastAggregator() { - DoubleLastAggregator agg = (DoubleLastAggregator) doubleLastAggFactory.factorize(colSelectorFactory); + Aggregator agg = doubleLastAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { - DoubleLastBufferAggregator agg = (DoubleLastBufferAggregator) doubleLastAggFactory.factorizeBuffered( + BufferAggregator agg = doubleLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleLastAggFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleLastCombiningAggregator() { - DoubleLastAggregator agg = (DoubleLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { - DoubleLastBufferAggregator agg = (DoubleLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[doubleLastAggFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - DoubleLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - DoubleLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java index 1938f8387339..2179f8feb809 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestFloatColumnSelector; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; @@ -71,7 +73,7 @@ public void setup() @Test public void testDoubleLastAggregator() { - FloatLastAggregator agg = (FloatLastAggregator) floatLastAggregatorFactory.factorize(colSelectorFactory); + Aggregator agg = floatLastAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -89,7 +91,7 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { - FloatLastBufferAggregator agg = (FloatLastBufferAggregator) floatLastAggregatorFactory.factorizeBuffered( + BufferAggregator agg = floatLastAggregatorFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSize()]); @@ -119,7 +121,7 @@ public void testCombine() @Test public void testDoubleLastCombiningAggregator() { - FloatLastAggregator agg = (FloatLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -138,7 +140,7 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { - FloatLastBufferAggregator agg = (FloatLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSize()]); @@ -168,7 +170,7 @@ public void testSerde() throws Exception } private void aggregate( - FloatLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -178,7 +180,7 @@ private void aggregate( } private void aggregate( - FloatLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java index 25d9160e15f7..5106749298c8 100644 --- a/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/last/LongLastAggregationTest.java @@ -22,7 +22,9 @@ import io.druid.collections.SerializablePair; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Pair; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; import io.druid.query.aggregation.TestLongColumnSelector; import io.druid.query.aggregation.TestObjectColumnSelector; import io.druid.segment.ColumnSelectorFactory; @@ -70,7 +72,7 @@ public void setup() @Test public void testLongLastAggregator() { - LongLastAggregator agg = (LongLastAggregator) longLastAggFactory.factorize(colSelectorFactory); + Aggregator agg = longLastAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -88,7 +90,7 @@ public void testLongLastAggregator() @Test public void testLongLastBufferAggregator() { - LongLastBufferAggregator agg = (LongLastBufferAggregator) longLastAggFactory.factorizeBuffered( + BufferAggregator agg = longLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longLastAggFactory.getMaxIntermediateSize()]); @@ -118,7 +120,7 @@ public void testCombine() @Test public void testLongLastCombiningAggregator() { - LongLastAggregator agg = (LongLastAggregator) combiningAggFactory.factorize(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); aggregate(agg); @@ -137,7 +139,7 @@ public void testLongLastCombiningAggregator() @Test public void testLongLastCombiningBufferAggregator() { - LongLastBufferAggregator agg = (LongLastBufferAggregator) combiningAggFactory.factorizeBuffered( + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[longLastAggFactory.getMaxIntermediateSize()]); @@ -167,7 +169,7 @@ public void testSerde() throws Exception } private void aggregate( - LongLastAggregator agg + Aggregator agg ) { agg.aggregate(); @@ -177,7 +179,7 @@ private void aggregate( } private void aggregate( - LongLastBufferAggregator agg, + BufferAggregator agg, ByteBuffer buff, int position ) diff --git a/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java b/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java index 9ad874985a00..e4515852ab0b 100644 --- a/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java +++ b/processing/src/test/java/io/druid/query/cache/CacheKeyBuilderTest.java @@ -24,8 +24,8 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; -import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.Cacheable; +import io.druid.java.util.common.StringUtils; import org.junit.Test; import java.nio.ByteBuffer; diff --git a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java index 609e74f375d4..078940ccf93f 100644 --- a/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java +++ b/processing/src/test/java/io/druid/query/extraction/FunctionalExtractionTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -135,7 +136,7 @@ public void testRetainMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -149,7 +150,7 @@ public void testRetainMissingButFound() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? in : out, exFn.apply(in)); + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? in : out, exFn.apply(in)); } @Test @@ -163,7 +164,11 @@ public void testReplaceMissing() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? MISSING : out, exFn.apply(in)); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(NullHandling.isNullOrEquivalent(out) ? MISSING : out, exFn.apply(in)); + } else { + Assert.assertEquals(out == null ? MISSING : out, exFn.apply(in)); + } } @@ -178,7 +183,11 @@ public void testReplaceMissingBlank() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + } else { + Assert.assertEquals(out == null ? "" : out, exFn.apply(in)); + } } @Test @@ -192,7 +201,11 @@ public void testOnlyOneValuePresent() false ); final String out = fn.apply(in); - Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? null : out, exFn.apply(in)); + } else { + Assert.assertEquals(Strings.isNullOrEmpty(out) ? "" : out, exFn.apply(in)); + } } @Test @@ -204,7 +217,7 @@ public void testNullInputs() null, false ); - if (Strings.isNullOrEmpty(fn.apply(null))) { + if (NullHandling.isNullOrEquivalent(fn.apply(null))) { Assert.assertEquals(null, exFn.apply(null)); } } diff --git a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java index dc4494dadbde..3672adea16de 100644 --- a/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/JavaScriptExtractionFnTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import io.druid.js.JavaScriptConfig; @@ -131,7 +132,11 @@ public void testJavascriptIsNull() Assert.assertEquals("yes", extractionFn.apply((String) null)); Assert.assertEquals("yes", extractionFn.apply((Object) null)); - Assert.assertEquals("yes", extractionFn.apply("")); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals("yes", extractionFn.apply("")); + } else { + Assert.assertEquals("no", extractionFn.apply("")); + } Assert.assertEquals("no", extractionFn.apply("abc")); Assert.assertEquals("no", extractionFn.apply(new Object())); Assert.assertEquals("no", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java index 1a7727adbaf1..3bd8f5464dc3 100644 --- a/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/LowerExtractionFnTest.java @@ -19,6 +19,7 @@ package io.druid.query.extraction; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,7 @@ public class LowerExtractionFnTest public void testApply() { Assert.assertEquals("lower 1 string", extractionFn.apply("lOwER 1 String")); - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java index 4415056f542a..839e9f5f838c 100644 --- a/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/MatchingDimExtractionFnTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -75,7 +76,7 @@ public void testNullExtraction() Assert.assertNull(extractionFn.apply((Object) null)); Assert.assertNull(extractionFn.apply((String) null)); - Assert.assertNull(extractionFn.apply((String) "")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply((String) "")); } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java index ce6e468ccd93..465036df30e1 100644 --- a/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/RegexDimExtractionFnTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -145,11 +146,11 @@ public void testNullAndEmpty() String regex = "(.*)/.*/.*"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, false, null); // no match, map empty input value to null - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); // null value, returns null Assert.assertEquals(null, extractionFn.apply(null)); // empty match, map empty result to null - Assert.assertEquals(null, extractionFn.apply("/a/b")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("/a/b")); } @Test @@ -168,8 +169,8 @@ public void testMissingValueReplacementWhenPatternMatchesNull() { String regex = "^()$"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, "NO MATCH"); - Assert.assertEquals(null, extractionFn.apply("")); - Assert.assertEquals(null, extractionFn.apply(null)); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "NO MATCH", extractionFn.apply(null)); Assert.assertEquals("NO MATCH", extractionFn.apply("abc")); } @@ -178,10 +179,10 @@ public void testMissingValueReplacementToEmpty() { String regex = "(bob)"; ExtractionFn extractionFn = new RegexDimExtractionFn(regex, true, ""); - Assert.assertEquals(null, extractionFn.apply(null)); - Assert.assertEquals(null, extractionFn.apply("")); - Assert.assertEquals(null, extractionFn.apply("abc")); - Assert.assertEquals(null, extractionFn.apply("123")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply(null)); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("abc")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("123")); Assert.assertEquals("bob", extractionFn.apply("bobby")); } diff --git a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java index 08b102c6cfb2..ac4ad42e630a 100644 --- a/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/StrlenExtractionFnTest.java @@ -20,6 +20,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -29,7 +30,7 @@ public class StrlenExtractionFnTest @Test public void testApply() { - Assert.assertEquals("0", StrlenExtractionFn.instance().apply(null)); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? "0" : null, StrlenExtractionFn.instance().apply(null)); Assert.assertEquals("0", StrlenExtractionFn.instance().apply("")); Assert.assertEquals("1", StrlenExtractionFn.instance().apply("x")); Assert.assertEquals("3", StrlenExtractionFn.instance().apply("foo")); diff --git a/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java index 77fb05cdb2e4..0be208d35083 100644 --- a/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/TimeDimExtractionFnTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; +import io.druid.common.config.NullHandling; import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -46,7 +47,11 @@ public void testEmptyAndNullExtraction() ExtractionFn extractionFn = new TimeDimExtractionFn("MM/dd/yyyy", "MM/yyyy"); Assert.assertNull(extractionFn.apply(null)); - Assert.assertNull(extractionFn.apply("")); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertNull(extractionFn.apply("")); + } else { + Assert.assertEquals("", extractionFn.apply("")); + } } @Test diff --git a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java index 1d038c0674ac..8b852822d2d4 100644 --- a/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/UpperExtractionFnTest.java @@ -19,6 +19,7 @@ package io.druid.query.extraction; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -31,7 +32,7 @@ public class UpperExtractionFnTest public void testApply() { Assert.assertEquals("UPPER", extractionFn.apply("uPpeR")); - Assert.assertEquals(null, extractionFn.apply("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? null : "", extractionFn.apply("")); Assert.assertEquals(null, extractionFn.apply(null)); Assert.assertEquals(null, extractionFn.apply((Object) null)); Assert.assertEquals("1", extractionFn.apply(1)); diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java index a27cf5a03482..9d080e754c56 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java @@ -34,7 +34,6 @@ import io.druid.collections.DefaultBlockingPool; import io.druid.collections.NonBlockingPool; import io.druid.collections.StupidPool; -import io.druid.java.util.common.concurrent.Execs; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.Row; @@ -43,6 +42,7 @@ import io.druid.data.input.impl.StringDimensionSchema; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 0eda36786656..65fd5274288c 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -35,6 +35,7 @@ import io.druid.collections.DefaultBlockingPool; import io.druid.collections.NonBlockingPool; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; @@ -6949,12 +6950,23 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() .setGranularity(QueryRunnerTestHelper.dayGran) .setDimFilter(new ExtractionDimFilter("quality", "", lookupExtractionFn, null)) .build(); - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) - ); + + List expectedResults; + + if (NullHandling.useDefaultValuesForNull()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + ); + } else { + // Only empty string should match, nulls will not match + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); @@ -7002,11 +7014,19 @@ public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() @Test public void testGroupByWithExtractionDimFilterKeyisNull() { + Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULLorEMPTY"); + MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.useDefaultValuesForNull()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + extractionMap.put("", "REPLACED_VALUE"); + } else { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "REPLACED_VALUE", true, false); + extractionMap.put("", "NOT_USED"); + } GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -7028,7 +7048,7 @@ public void testGroupByWithExtractionDimFilterKeyisNull() .setDimFilter( new ExtractionDimFilter( "null_column", - "NULLorEMPTY", + "REPLACED_VALUE", lookupExtractionFn, null ) @@ -7084,25 +7104,137 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() .setGranularity(QueryRunnerTestHelper.dayGran) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "automotive", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "business", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "entertainment", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "health", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 0L, "idx", 0L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "premium", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "technology", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "alias", + "travel", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "automotive", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "business", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "entertainment", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "health", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 0L, "idx", 0L) + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "premium", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "technology", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-02", + "alias", + "travel", + "rows", + 0L, + "idx", + NullHandling.useDefaultValuesForNull() ? 0L : null + ) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -7157,7 +7289,14 @@ public void testGroupByWithExtractionDimFilterNullDims() extractionMap.put("", "EMPTY"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.useDefaultValuesForNull()) { + extractionMap.put("", "EMPTY"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + } else { + extractionMap.put("", "SHOULD_NOT_BE_USED"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "EMPTY", true, true); + } GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) @@ -8133,23 +8272,42 @@ public void testGroupByNumericStringsAsNumericWithDecoration() ) ) .setGranularity(QueryRunnerTestHelper.allGran) + .addOrderByColumn("ql") .build(); - // "entertainment" rows are excluded by the decorated specs, they become empty rows - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 0L, - "qf", 0.0, - "count", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 170000L, - "qf", 170000.0, - "count", 2L - ) - ); + List expectedResults; + if (NullHandling.useDefaultValuesForNull()) { + // "entertainment" rows are excluded by the decorated specs, they become empty rows + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 0L, + "qf", 0.0, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 170000L, + "qf", 170000.0, + "count", 2L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", null, + "qf", null, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 170000L, + "qf", 170000.0, + "count", 2L + ) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); @@ -8192,21 +8350,38 @@ public void testGroupByDecorationOnNumerics() ) .setGranularity(QueryRunnerTestHelper.allGran) .build(); - - List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 0L, - "qf", 0.0, - "count", 2L - ), - GroupByQueryRunnerTestHelper.createExpectedRow( - "2011-04-01", - "ql", 1700L, - "qf", 17000.0, - "count", 2L - ) - ); + List expectedResults; + if (NullHandling.useDefaultValuesForNull()) { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 0L, + "qf", 0.0, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 1700L, + "qf", 17000.0, + "count", 2L + ) + ); + } else { + expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", null, + "qf", null, + "count", 2L + ), + GroupByQueryRunnerTestHelper.createExpectedRow( + "2011-04-01", + "ql", 1700L, + "qf", 17000.0, + "count", 2L + ) + ); + } Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java index d3bd541a66b2..2439e4a49743 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Ordering; import com.google.common.io.Files; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.ByteBufferUtils; import io.druid.query.aggregation.AggregatorFactory; @@ -111,7 +112,7 @@ public void testGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, 2); - final int expectedMaxSize = 219; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 219 : 210; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -139,7 +140,7 @@ public void testGrowing2() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 2_000_000_000, 2); - final int expectedMaxSize = 40988516; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 40988516 : 39141224; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -153,7 +154,7 @@ public void testGrowing3() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, Integer.MAX_VALUE, 2); - final int expectedMaxSize = 44938972; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 44938972 : 42955456; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { @@ -167,7 +168,7 @@ public void testNoGrowing() { final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final Grouper grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE); - final int expectedMaxSize = 267; + final int expectedMaxSize = NullHandling.useDefaultValuesForNull() ? 267 : 258; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); for (int i = 0; i < expectedMaxSize; i++) { diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java index 6710b74bd8fc..7caab84c4f69 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/LimitedBufferHashGrouperTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.IAE; import io.druid.query.aggregation.AggregatorFactory; @@ -53,20 +54,37 @@ public void testLimitAndBufferSwapping() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i + keyBase), grouper.aggregate(i + keyBase).isOk()); } + if (NullHandling.useDefaultValuesForNull()) { + // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes + // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes + // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes + // table arena is split in halves when doing push down, so each half is 9798 bytes + // each table arena half can hold 9798 / 28 = 349 buckets, with load factor of 0.5 max buckets per half is 174 + // First buffer swap occurs when we hit 174 buckets + // Subsequent buffer swaps occur after every 74 buckets, since we keep 100 buckets due to the limit + // With 1000 keys inserted, this results in one swap at the first 174 buckets, then 11 swaps afterwards. + // After the last swap, we have 100 keys + 12 new keys inserted. + Assert.assertEquals(12, grouper.getGrowthCount()); + Assert.assertEquals(112, grouper.getSize()); + Assert.assertEquals(349, grouper.getBuckets()); + Assert.assertEquals(174, grouper.getMaxSize()); + } else { + // With Nullability enabled + // bucket size is hash(int) + key(int) + aggs(2 longs + 1 bytes for Long Agg nullability) + heap offset(int) = 29 bytes + // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes + // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes + // table arena is split in halves when doing push down, so each half is 9798 bytes + // each table arena half can hold 9798 / 29 = 337 buckets, with load factor of 0.5 max buckets per half is 168 + // First buffer swap occurs when we hit 168 buckets + // Subsequent buffer swaps occur after every 68 buckets, since we keep 100 buckets due to the limit + // With 1000 keys inserted, this results in one swap at the first 169 buckets, then 12 swaps afterwards. + // After the last swap, we have 100 keys + 16 new keys inserted. + Assert.assertEquals(13, grouper.getGrowthCount()); + Assert.assertEquals(116, grouper.getSize()); + Assert.assertEquals(337, grouper.getBuckets()); + Assert.assertEquals(168, grouper.getMaxSize()); + } - // bucket size is hash(int) + key(int) + aggs(2 longs) + heap offset(int) = 28 bytes - // limit is 100 so heap occupies 101 * 4 bytes = 404 bytes - // buffer is 20000 bytes, so table arena size is 20000 - 404 = 19596 bytes - // table arena is split in halves when doing push down, so each half is 9798 bytes - // each table arena half can hold 9798 / 28 = 349 buckets, with load factor of 0.5 max buckets per half is 174 - // First buffer swap occurs when we hit 174 buckets - // Subsequent buffer swaps occur after every 74 buckets, since we keep 100 buckets due to the limit - // With 1000 keys inserted, this results in one swap at the first 174 buckets, then 11 swaps afterwards. - // After the last swap, we have 100 keys + 12 new keys inserted. - Assert.assertEquals(12, grouper.getGrowthCount()); - Assert.assertEquals(112, grouper.getSize()); - Assert.assertEquals(349, grouper.getBuckets()); - Assert.assertEquals(174, grouper.getMaxSize()); Assert.assertEquals(100, grouper.getLimit()); // Aggregate slightly different row @@ -77,14 +95,27 @@ public void testLimitAndBufferSwapping() Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - // we added another 1000 unique keys - // previous size is 112, so next swap occurs after 62 rows - // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, - // with 50 keys being added after the final swap. - Assert.assertEquals(25, grouper.getGrowthCount()); - Assert.assertEquals(150, grouper.getSize()); - Assert.assertEquals(349, grouper.getBuckets()); - Assert.assertEquals(174, grouper.getMaxSize()); + if (NullHandling.useDefaultValuesForNull()) { + // we added another 1000 unique keys + // previous size is 112, so next swap occurs after 62 rows + // after that, there are 1000 - 62 = 938 rows, 938 / 74 = 12 additional swaps after the first, + // with 50 keys being added after the final swap. + Assert.assertEquals(25, grouper.getGrowthCount()); + Assert.assertEquals(150, grouper.getSize()); + Assert.assertEquals(349, grouper.getBuckets()); + Assert.assertEquals(174, grouper.getMaxSize()); + } else { + // With Nullable Aggregator + // we added another 1000 unique keys + // previous size is 116, so next swap occurs after 52 rows + // after that, there are 1000 - 52 = 948 rows, 948 / 68 = 13 additional swaps after the first, + // with 64 keys being added after the final swap. + Assert.assertEquals(27, grouper.getGrowthCount()); + Assert.assertEquals(164, grouper.getSize()); + Assert.assertEquals(337, grouper.getBuckets()); + Assert.assertEquals(168, grouper.getMaxSize()); + } + Assert.assertEquals(100, grouper.getLimit()); final List> expected = Lists.newArrayList(); @@ -110,7 +141,7 @@ public void testMinBufferSize() final int limit = 100; final int keyBase = 100000; final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); - final LimitedBufferHashGrouper grouper = makeGrouper(columnSelectorFactory, 11716, 2, limit); + final LimitedBufferHashGrouper grouper = makeGrouper(columnSelectorFactory, 12120, 2, limit); final int numRows = 1000; columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); @@ -119,10 +150,17 @@ public void testMinBufferSize() } // With minimum buffer size, after the first swap, every new key added will result in a swap - Assert.assertEquals(899, grouper.getGrowthCount()); - Assert.assertEquals(101, grouper.getSize()); - Assert.assertEquals(202, grouper.getBuckets()); - Assert.assertEquals(101, grouper.getMaxSize()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(224, grouper.getGrowthCount()); + Assert.assertEquals(104, grouper.getSize()); + Assert.assertEquals(209, grouper.getBuckets()); + Assert.assertEquals(104, grouper.getMaxSize()); + } else { + Assert.assertEquals(899, grouper.getGrowthCount()); + Assert.assertEquals(101, grouper.getSize()); + Assert.assertEquals(202, grouper.getBuckets()); + Assert.assertEquals(101, grouper.getMaxSize()); + } Assert.assertEquals(100, grouper.getLimit()); // Aggregate slightly different row @@ -132,11 +170,17 @@ public void testMinBufferSize() for (int i = 0; i < numRows; i++) { Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk()); } - - Assert.assertEquals(1899, grouper.getGrowthCount()); - Assert.assertEquals(101, grouper.getSize()); - Assert.assertEquals(202, grouper.getBuckets()); - Assert.assertEquals(101, grouper.getMaxSize()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(474, grouper.getGrowthCount()); + Assert.assertEquals(104, grouper.getSize()); + Assert.assertEquals(209, grouper.getBuckets()); + Assert.assertEquals(104, grouper.getMaxSize()); + } else { + Assert.assertEquals(1899, grouper.getGrowthCount()); + Assert.assertEquals(101, grouper.getSize()); + Assert.assertEquals(202, grouper.getBuckets()); + Assert.assertEquals(101, grouper.getMaxSize()); + } Assert.assertEquals(100, grouper.getLimit()); final List> expected = Lists.newArrayList(); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java index e66cd77b0ac5..00710351c0a7 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/StreamingMergeSortedGrouperTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedRow; import io.druid.java.util.common.concurrent.Execs; import io.druid.query.aggregation.AggregatorFactory; @@ -100,7 +101,7 @@ public void testStreamingAggregateWithLargeBuffer() throws ExecutionException, I @Test(timeout = 5000L) public void testStreamingAggregateWithMinimumBuffer() throws ExecutionException, InterruptedException { - testStreamingAggregate(60); + testStreamingAggregate(83); } private void testStreamingAggregate(int bufferSize) throws ExecutionException, InterruptedException @@ -128,7 +129,10 @@ private void testStreamingAggregate(int bufferSize) throws ExecutionException, I }); final List> unsortedEntries = Lists.newArrayList(grouper.iterator(true)); - final List> actual = Ordering.from((Comparator>) (o1, o2) -> Ints.compare(o1.getKey(), o2.getKey())) + final List> actual = Ordering.from((Comparator>) (o1, o2) -> Ints.compare( + o1.getKey(), + o2.getKey() + )) .sortedCopy(unsortedEntries); if (!actual.equals(expected)) { @@ -145,7 +149,11 @@ private void testStreamingAggregate(int bufferSize) throws ExecutionException, I public void testNotEnoughBuffer() { expectedException.expect(IllegalStateException.class); - expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[20]"); + if (NullHandling.useDefaultValuesForNull()) { + expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[20]"); + } else { + expectedException.expectMessage("Buffer[50] should be large enough to store at least three records[21]"); + } newGrouper(GrouperTestUtil.newColumnSelectorFactory(), 50); } @@ -157,7 +165,7 @@ public void testTimeout() expectedException.expectCause(CoreMatchers.instanceOf(TimeoutException.class)); final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); - final StreamingMergeSortedGrouper grouper = newGrouper(columnSelectorFactory, 60); + final StreamingMergeSortedGrouper grouper = newGrouper(columnSelectorFactory, 100); columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L))); grouper.aggregate(6); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java index 8428981da063..1c05fd68825c 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java @@ -83,8 +83,16 @@ public Class classOfObject() @Override public void inspectRuntimeShape(RuntimeShapeInspector inspector) { - // don't inspect in tests + // Nothing to do. } + + @Override + public boolean isNull() + { + return row.get().getMetric(columnName) == null; + } + + }; } diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java index a363f7606039..4972e3c6130a 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnExpectationsTest.java @@ -20,6 +20,7 @@ package io.druid.query.lookup; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.query.extraction.MapLookupExtractor; import org.junit.Assert; import org.junit.Test; @@ -65,7 +66,11 @@ public void testNullKeyIsMappable() false, false ); - Assert.assertEquals("bar", lookupExtractionFn.apply(null)); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals("bar", lookupExtractionFn.apply(null)); + } else { + Assert.assertEquals("REPLACE", lookupExtractionFn.apply(null)); + } } @Test diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java index d60f728764e9..c052e3d461fd 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java @@ -74,7 +74,9 @@ public Object[] apply(List input) public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, boolean injective) { + //CHECKSTYLE.OFF: Regexp this.replaceMissing = Strings.emptyToNull(replaceMissing); + //CHECKSTYLE.ON: Regexp this.retainMissing = retainMissing; this.injective = injective; } @@ -82,7 +84,9 @@ public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, bool @Test public void testEqualsAndHash() { + //CHECKSTYLE.OFF: Regexp if (retainMissing && !Strings.isNullOrEmpty(replaceMissing)) { + //CHECKSTYLE.ON: Regexp // skip return; } @@ -154,7 +158,9 @@ public void testIllegalArgs() final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( new MapLookupExtractor(ImmutableMap.of("foo", "bar"), false), retainMissing, + //CHECKSTYLE.OFF: Regexp Strings.emptyToNull(replaceMissing), + //CHECKSTYLE.ON: Regexp injective, false ); diff --git a/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java b/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java index 4b7153dc6ac8..f589de76deae 100644 --- a/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/scan/ScanQueryRunnerTest.java @@ -693,7 +693,12 @@ public static void verify( Object exValue = ex.getValue(); if (exValue instanceof Double || exValue instanceof Float) { final double expectedDoubleValue = ((Number) exValue).doubleValue(); - Assert.assertEquals("invalid value for " + ex.getKey(), expectedDoubleValue, ((Number) actVal).doubleValue(), expectedDoubleValue * 1e-6); + Assert.assertEquals( + "invalid value for " + ex.getKey(), + expectedDoubleValue, + ((Number) actVal).doubleValue(), + expectedDoubleValue * 1e-6 + ); } else { Assert.assertEquals("invalid value for " + ex.getKey(), ex.getValue(), actVal); } diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index d53c55396dfd..6e9fd0ab6ebf 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -752,7 +753,11 @@ public void testSearchWithNullValueInDimension() throws Exception QueryRunner runner = factory.createRunner(new QueryableIndexSegment("asdf", TestIndex.persistRealtimeAndLoadMMapped(index))); List expectedHits = Lists.newLinkedList(); expectedHits.add(new SearchHit("table", "table", 1)); - expectedHits.add(new SearchHit("table", "", 1)); + if (NullHandling.useDefaultValuesForNull()) { + expectedHits.add(new SearchHit("table", "", 1)); + } else { + expectedHits.add(new SearchHit("table", null, 1)); + } checkSearchQuery(searchQuery, runner, expectedHits); } diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index a397f5c3ba63..505d0ee907d8 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -23,7 +23,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.primitives.Doubles; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; @@ -144,16 +146,15 @@ public void testEmptyTimeseries() ) .descending(descending) .build(); - + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0D : null); + resultMap.put("first", NullHandling.useDefaultValuesForNull() ? 0D : null); List> expectedResults = ImmutableList.of( new Result<>( DateTimes.of("2020-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0D, - "first", 0D - ) + resultMap ) ) ); @@ -205,24 +206,61 @@ public void testFullOnTimeseries() QueryRunnerTestHelper.skippedDay.equals(current) ? 0L : 13L, value.getLongMetric("rows").longValue() ); - Assert.assertEquals( - result.toString(), - Doubles.tryParse(expectedIndex[count]).doubleValue(), - value.getDoubleMetric("index").doubleValue(), - value.getDoubleMetric("index").doubleValue() * 1e-6 - ); - Assert.assertEquals( - result.toString(), - new Double(expectedIndex[count]) + - (QueryRunnerTestHelper.skippedDay.equals(current) ? 0L : 13L) + 1L, - value.getDoubleMetric("addRowsIndexConstant"), - value.getDoubleMetric("addRowsIndexConstant") * 1e-6 - ); - Assert.assertEquals( - value.getDoubleMetric("uniques"), - QueryRunnerTestHelper.skippedDay.equals(current) ? 0.0d : 9.0d, - 0.02 - ); + + if (!QueryRunnerTestHelper.skippedDay.equals(current)) { + Assert.assertEquals( + result.toString(), + Doubles.tryParse(expectedIndex[count]).doubleValue(), + value.getDoubleMetric("index").doubleValue(), + value.getDoubleMetric("index").doubleValue() * 1e-6 + ); + Assert.assertEquals( + result.toString(), + new Double(expectedIndex[count]) + + 13L + 1L, + value.getDoubleMetric("addRowsIndexConstant"), + value.getDoubleMetric("addRowsIndexConstant") * 1e-6 + ); + Assert.assertEquals( + value.getDoubleMetric("uniques"), + 9.0d, + 0.02 + ); + } else { + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals( + result.toString(), + 0.0D, + value.getDoubleMetric("index").doubleValue(), + value.getDoubleMetric("index").doubleValue() * 1e-6 + ); + Assert.assertEquals( + result.toString(), + new Double(expectedIndex[count]) + 1L, + value.getDoubleMetric("addRowsIndexConstant"), + value.getDoubleMetric("addRowsIndexConstant") * 1e-6 + ); + Assert.assertEquals( + 0.0D, + value.getDoubleMetric("uniques"), + 0.02 + ); + } else { + Assert.assertNull( + result.toString(), + value.getDoubleMetric("index") + ); + Assert.assertNull( + result.toString(), + value.getDoubleMetric("addRowsIndexConstant") + ); + Assert.assertEquals( + value.getDoubleMetric("uniques"), + 0.0d, + 0.02 + ); + } + } lastResult = result; ++count; @@ -626,14 +664,15 @@ public void testTimeseriesQueryZeroFilling() final Iterable iterable = Granularities.HOUR.getIterable( new Interval(DateTimes.of("2011-04-14T01"), DateTimes.of("2011-04-15")) ); + Map noRowsResult = Maps.newHashMap(); + noRowsResult.put("rows", 0L); + noRowsResult.put("idx", NullHandling.useDefaultValuesForNull() ? 0L : null); for (Interval interval : iterable) { lotsOfZeroes.add( - new Result<>( - interval.getStart(), - new TimeseriesResultValue( - ImmutableMap.of("rows", 0L, "idx", 0L) - ) - ) + new Result<>( + interval.getStart(), + new TimeseriesResultValue(noRowsResult) + ) ); } @@ -1338,27 +1377,23 @@ public void testTimeseriesWithFilterOnNonExistentDimension() .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("uniques", 0.0); + List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -1483,28 +1518,23 @@ public void testTimeseriesWithNonExistentFilter() .postAggregators(QueryRunnerTestHelper.addRowsIndexConstant) .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -1529,28 +1559,23 @@ public void testTimeseriesWithNonExistentFilterAndMultiDim() .postAggregators(QueryRunnerTestHelper.addRowsIndexConstant) .descending(descending) .build(); + Map resultMap = Maps.newHashMap(); + resultMap.put("rows", 0L); + resultMap.put("index", NullHandling.useDefaultValuesForNull() ? 0.0 : null); + resultMap.put("addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0 : null); + resultMap.put("uniques", 0.0); List> expectedResults = Arrays.asList( new Result<>( DateTimes.of("2011-04-01"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ), new Result<>( DateTimes.of("2011-04-02"), new TimeseriesResultValue( - ImmutableMap.of( - "rows", 0L, - "index", 0.0, - "addRowsIndexConstant", 1.0, - "uniques", 0.0 - ) + resultMap ) ) ); @@ -2220,7 +2245,11 @@ public void testTimeSeriesWithSelectionFilterLookupExtractionFn() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.dayGran) .filters( - new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "upfront", lookupExtractionFn) + new SelectorDimFilter( + QueryRunnerTestHelper.marketDimension, + "upfront", + lookupExtractionFn + ) ) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index 1185c8044d3f..27807a738ef4 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -29,6 +29,7 @@ import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; @@ -4206,10 +4207,15 @@ public void testTopNWithExtractionFilter() public void testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULL"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.useDefaultValuesForNull()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); + extractionMap.put("", "NULL"); + } else { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, false); + } DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) @@ -4279,10 +4285,16 @@ private Sequence> runWithPreMergeAndMerge(TopNQuery quer public void testTopNWithExtractionFilterNoExistingValue() { Map extractionMap = new HashMap<>(); - extractionMap.put("", "NULL"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + LookupExtractionFn lookupExtractionFn; + if (NullHandling.useDefaultValuesForNull()) { + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); + extractionMap.put("", "NULL"); + } else { + extractionMap.put("", "NOT_USED"); + lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, true); + } DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null); TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) diff --git a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java index abac95cf0159..1202159ab7a7 100644 --- a/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/ConstantDimensionSelectorTest.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.common.config.NullHandling; import io.druid.query.extraction.StringFormatExtractionFn; import io.druid.query.extraction.SubstringDimExtractionFn; import io.druid.segment.data.IndexedInts; @@ -68,7 +69,7 @@ public void testLookupName() throws Exception public void testLookupId() throws Exception { Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId(null)); - Assert.assertEquals(0, NULL_SELECTOR.idLookup().lookupId("")); + Assert.assertEquals(NullHandling.useDefaultValuesForNull() ? 0 : -1, NULL_SELECTOR.idLookup().lookupId("")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("billy")); Assert.assertEquals(-1, NULL_SELECTOR.idLookup().lookupId("bob")); diff --git a/processing/src/test/java/io/druid/segment/IndexIOTest.java b/processing/src/test/java/io/druid/segment/IndexIOTest.java index 4fefaea23555..a50704120589 100644 --- a/processing/src/test/java/io/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/io/druid/segment/IndexIOTest.java @@ -205,7 +205,7 @@ public Object[] next() maplist1, maplist2, filterNullValues(maplist1).equals(filterNullValues(maplist2)) ? - null : SegmentValidationException.class + null : SegmentValidationException.class }; } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java index 278a1d759098..4bfb68dad2cc 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerNullHandlingTest.java @@ -19,12 +19,12 @@ package io.druid.segment; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultiset; import com.google.common.collect.Sets; import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Comparators; @@ -89,9 +89,15 @@ public void testStringColumnNullHandling() throws Exception nullFlavors.add(mMissing); nullFlavors.add(mEmptyList); nullFlavors.add(mNull); - nullFlavors.add(mEmptyString); nullFlavors.add(mListOfNull); - nullFlavors.add(mListOfEmptyString); + + if (NullHandling.useDefaultValuesForNull()) { + nullFlavors.add(mEmptyString); + nullFlavors.add(mListOfEmptyString); + } else { + nonNullFlavors.add(mEmptyString); + nonNullFlavors.add(mListOfEmptyString); + } Set> allValues = new HashSet<>(); allValues.addAll(nonNullFlavors); @@ -147,7 +153,13 @@ public void testStringColumnNullHandling() throws Exception Assert.assertEquals(subsetList.toString(), hasMultipleValues, dictionaryColumn.hasMultipleValues()); Assert.assertEquals(subsetList.toString(), uniqueValues.size(), dictionaryColumn.getCardinality()); - + System.out.println(hasMultipleValues); + System.out.println(ImmutableMultiset.copyOf( + subsetList.stream() + .map(m -> normalize(m.get("d"), hasMultipleValues)) + .distinct() // Distinct values only, because we expect rollup. + .collect(Collectors.toList()) + )); // Verify the expected set of rows was indexed, ignoring order. Assert.assertEquals( subsetList.toString(), @@ -174,7 +186,7 @@ public void testStringColumnNullHandling() throws Exception final List expectedNullRows = new ArrayList<>(); for (int i = 0; i < index.getNumRows(); i++) { final List row = getRow(dictionaryColumn, i); - if (row.isEmpty() || row.stream().anyMatch(Strings::isNullOrEmpty)) { + if (row.isEmpty() || row.stream().anyMatch(NullHandling::isNullOrEquivalent)) { expectedNullRows.add(i); } } @@ -209,19 +221,16 @@ private static List normalize(final Object value, final boolean hasMulti final List retVal = new ArrayList<>(); if (value == null) { - if (!hasMultipleValues) { - // nulls become nulls in single valued columns, but are empty lists in multi valued columns - retVal.add(null); - } + retVal.add(null); } else if (value instanceof String) { - retVal.add(Strings.emptyToNull(((String) value))); + retVal.add(NullHandling.emptyToNullIfNeeded(((String) value))); } else if (value instanceof List) { final List list = (List) value; if (list.isEmpty() && !hasMultipleValues) { // empty lists become nulls in single valued columns - retVal.add(null); + retVal.add(NullHandling.emptyToNullIfNeeded(null)); } else { - retVal.addAll(list.stream().map(Strings::emptyToNull).collect(Collectors.toList())); + retVal.addAll(list.stream().map(NullHandling::emptyToNullIfNeeded).collect(Collectors.toList())); } } else { throw new ISE("didn't expect class[%s]", value.getClass()); diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index 725b69ffc2fe..c071da80c6f0 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -28,6 +28,7 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionSchema; @@ -41,7 +42,6 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; -import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.LongSumAggregatorFactory; @@ -58,6 +58,7 @@ import io.druid.segment.incremental.IncrementalIndexAdapter; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.IndexSizeExceededException; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import it.unimi.dsi.fastutil.ints.IntIterator; import org.joda.time.Interval; import org.junit.Assert; @@ -233,11 +234,11 @@ public void testPersistWithDifferentDims() throws Exception Assert.assertArrayEquals(new int[][]{{0}, {1}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{1}, {0}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim1", "3")); - checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", "")); + checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dim2", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim2", "2")); } @@ -902,17 +903,17 @@ public void testNonLexicographicDimOrderMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {1}, {1}}, boatList.get(2).getDims()); Assert.assertArrayEquals(new Object[]{3L}, boatList.get(2).getMetrics()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d3", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "30000")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "40000")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d3", "50000")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d1", "100")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d1", "200")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d1", "300")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d2", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d2", null)); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "2000")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d2", "3000")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d2", "4000")); @@ -1015,13 +1016,13 @@ public void testMergeWithDimensionsList() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList.get(3).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(3).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dimB", null)); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("dimC", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimC", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimC", "2")); } @@ -1118,11 +1119,11 @@ public void testDisjointDimMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList.get(4).getDims()); Assert.assertArrayEquals(new Object[]{1L}, boatList.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(3, 4), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(3, 4), adapter.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "3")); @@ -1140,11 +1141,11 @@ public void testDisjointDimMerge() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}}, boatList2.get(4).getDims()); Assert.assertArrayEquals(new Object[]{1L}, boatList2.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter2.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(3), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(4), adapter2.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(3, 4), adapter2.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimB", "3")); @@ -1251,40 +1252,60 @@ public void testJointDimMerge() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); - Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {1}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {2}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {1}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(1, 2, 3), adapter.getBitmapIndex("d1", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d5", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); + } - checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "210")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d5", "520")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d7", "720")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d8", "810")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d8", "820")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d9", "911")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d9", "920")); @@ -1396,40 +1417,64 @@ public void testNoRollupMergeWithoutDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals( + ImmutableList.of("d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } else { + Assert.assertEquals( + ImmutableList.of("d1", "d2", "d3", "d5", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + } Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {1}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + + checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + + } else { + // NULL and EMPTY Strings are considered different + Assert.assertArrayEquals(new int[][]{{1}, {1}, {1}, {0}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {2}, {2}, {0}, {0}, {1}, {1}, {1}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {2}, {1}, {2}, {2}, {2}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {0}, {0}, {1}, {2}, {0}, {3}, {3}}, boatList.get(3).getDims()); + + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d2", null)); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", null)); + checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", null)); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); + + } + - checkBitmapIndex(Lists.newArrayList(0, 2, 3), adapter.getBitmapIndex("d2", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d2", "210")); - checkBitmapIndex(Lists.newArrayList(2, 3), adapter.getBitmapIndex("d3", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d3", "310")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d3", "311")); - checkBitmapIndex(Lists.newArrayList(0, 1, 3), adapter.getBitmapIndex("d5", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d5", "520")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("d6", "")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d6", "620")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 3), adapter.getBitmapIndex("d7", "")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d7", "710")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d7", "720")); - checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", "")); + checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d8", null)); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d8", "810")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d8", "820")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("d9", "911")); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("d9", "920")); @@ -1461,7 +1506,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1470,7 +1515,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1485,7 +1530,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 1, Arrays.asList("d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d1", "", "d2", "", "d3", "310", "d7", "", "d9", "910" + "d3", "310", "d7", "", "d9", "910" ) ) ); @@ -1494,7 +1539,7 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception 4, Arrays.asList("d4", "d5", "d6", "d7", "d8", "d9"), ImmutableMap.of( - "d5", "", "d6", "621", "d7", "", "d8", "821", "d9", "921" + "d6", "621", "d7", "", "d8", "821", "d9", "921" ) ) ); @@ -1540,26 +1585,41 @@ public void testNoRollupMergeWithDuplicateRow() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - Assert.assertEquals( - ImmutableList.of("d3", "d6", "d8", "d9"), - ImmutableList.copyOf(adapter.getDimensionNames()) - ); - Assert.assertEquals(4, boatList.size()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(0).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(1).getDims()); - Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(2).getDims()); - Assert.assertArrayEquals(new int[][]{{0}, {1}, {1}, {1}}, boatList.get(3).getDims()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {1}, {1}}, boatList.get(3).getDims()); + } else { + Assert.assertEquals( + ImmutableList.of("d3", "d6", "d7", "d8", "d9"), + ImmutableList.copyOf(adapter.getDimensionNames()) + ); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new int[][]{{1}, {0}, {0}, {0}, {0}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new int[][]{{0}, {1}, {0}, {1}, {1}}, boatList.get(3).getDims()); + checkBitmapIndex(Lists.newArrayList(), adapter.getBitmapIndex("d7", null)); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3), adapter.getBitmapIndex("d7", "")); + } + - checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", "")); + checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d3", null)); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d3", "310")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d6", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d6", "621")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d8", null)); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d8", "821")); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("d9", null)); checkBitmapIndex(Lists.newArrayList(0, 1, 2), adapter.getBitmapIndex("d9", "910")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("d9", "921")); } @@ -1592,7 +1652,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "1", "dimA", "") + ImmutableMap.of("dimB", "1") ) ); @@ -1600,7 +1660,7 @@ public void testMergeWithSupersetOrdering() throws Exception new MapBasedInputRow( 1, Arrays.asList("dimB", "dimA"), - ImmutableMap.of("dimB", "", "dimA", "1") + ImmutableMap.of("dimA", "1") ) ); @@ -1716,11 +1776,11 @@ public void testMergeWithSupersetOrdering() throws Exception Assert.assertArrayEquals(new int[][]{{3}, {0}}, boatList.get(4).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList.get(4).getMetrics()); - checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(2, 3, 4), adapter.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(2), adapter.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(3), adapter.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(4), adapter.getBitmapIndex("dimB", "3")); @@ -1756,16 +1816,16 @@ public void testMergeWithSupersetOrdering() throws Exception Assert.assertArrayEquals(new int[][]{{2}, {0}, {0}}, boatList2.get(11).getDims()); Assert.assertArrayEquals(new Object[]{2L}, boatList2.get(11).getMetrics()); - checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 3, 4, 5, 8, 9, 10), adapter2.getBitmapIndex("dimA", null)); checkBitmapIndex(Lists.newArrayList(6), adapter2.getBitmapIndex("dimA", "1")); checkBitmapIndex(Lists.newArrayList(7, 11), adapter2.getBitmapIndex("dimA", "2")); - checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", "")); + checkBitmapIndex(Lists.newArrayList(0, 1, 2, 6, 7, 11), adapter2.getBitmapIndex("dimB", null)); checkBitmapIndex(Lists.newArrayList(3, 8), adapter2.getBitmapIndex("dimB", "1")); checkBitmapIndex(Lists.newArrayList(4, 9), adapter2.getBitmapIndex("dimB", "2")); checkBitmapIndex(Lists.newArrayList(5, 10), adapter2.getBitmapIndex("dimB", "3")); - checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", "")); + checkBitmapIndex(Lists.newArrayList(3, 4, 5, 6, 7, 8, 9, 10, 11), adapter2.getBitmapIndex("dimC", null)); checkBitmapIndex(Lists.newArrayList(0), adapter2.getBitmapIndex("dimC", "1")); checkBitmapIndex(Lists.newArrayList(1), adapter2.getBitmapIndex("dimC", "2")); checkBitmapIndex(Lists.newArrayList(2), adapter2.getBitmapIndex("dimC", "3")); @@ -2185,13 +2245,13 @@ public void testPersistNullColumnSkipping() throws Exception }); index1.add(new MapBasedInputRow( 1L, - Lists.newArrayList("d1", "d2"), + Lists.newArrayList("d1", "d2", "d3"), ImmutableMap.of("d1", "a", "d2", "", "A", 1) )); index1.add(new MapBasedInputRow( 1L, - Lists.newArrayList("d1", "d2"), + Lists.newArrayList("d1", "d2", "d3"), ImmutableMap.of("d1", "b", "d2", "", "A", 1) )); @@ -2206,14 +2266,30 @@ public void testPersistNullColumnSkipping() throws Exception ) ) ); - List expectedColumnNames = Arrays.asList("A", "d1"); + List expectedColumnNames = NullHandling.useDefaultValuesForNull() + ? Arrays.asList("A", "d1") + : Arrays.asList("A", "d1", "d2"); List actualColumnNames = Lists.newArrayList(index.getColumnNames()); Collections.sort(expectedColumnNames); Collections.sort(actualColumnNames); Assert.assertEquals(expectedColumnNames, actualColumnNames); SmooshedFileMapper sfm = closer.closeLater(SmooshedFileMapper.load(tempDir)); - List expectedFilenames = Arrays.asList("A", "__time", "d1", "index.drd", "metadata.drd"); + List expectedFilenames = NullHandling.useDefaultValuesForNull() ? Arrays.asList( + "A", + "__time", + "d1", + "index.drd", + "metadata.drd" + ) + : Arrays.asList( + "A", + "__time", + "d1", + "d2", + "index.drd", + "metadata.drd" + ); List actualFilenames = new ArrayList<>(sfm.getInternalFilenames()); Collections.sort(expectedFilenames); Collections.sort(actualFilenames); @@ -2398,7 +2474,7 @@ public void testMultiValueHandling() throws Exception Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {0, 1, 2}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{0, 0, 1, 2}, {0, 1, 2, 2}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); @@ -2421,7 +2497,7 @@ public void testMultiValueHandling() throws Exception Assert.assertEquals(1, boatList.size()); Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {0, 1, 2}}, boatList.get(0).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0), adapter.getBitmapIndex("dim1", "x")); @@ -2445,7 +2521,7 @@ public void testMultiValueHandling() throws Exception Assert.assertArrayEquals(new int[][]{{0, 1, 2}, {2, 0, 1}}, boatList.get(0).getDims()); Assert.assertArrayEquals(new int[][]{{2, 0, 0, 1}, {0, 2, 1, 2}}, boatList.get(1).getDims()); - checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", "")); + checkBitmapIndex(new ArrayList(), adapter.getBitmapIndex("dim1", null)); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "a")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "b")); checkBitmapIndex(Lists.newArrayList(0, 1), adapter.getBitmapIndex("dim1", "x")); diff --git a/processing/src/test/java/io/druid/segment/MetadataTest.java b/processing/src/test/java/io/druid/segment/MetadataTest.java index 3aba1dbc02d2..22901f74b175 100644 --- a/processing/src/test/java/io/druid/segment/MetadataTest.java +++ b/processing/src/test/java/io/druid/segment/MetadataTest.java @@ -45,7 +45,7 @@ public void testSerde() throws Exception Metadata metadata = new Metadata(); metadata.put("k", "v"); - AggregatorFactory[] aggregators = new AggregatorFactory[] { + AggregatorFactory[] aggregators = new AggregatorFactory[]{ new LongSumAggregatorFactory("out", "in") }; metadata.setAggregators(aggregators); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java index 83bf29cf20ab..85cc3b963afc 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestFullTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.Pair; @@ -400,13 +401,13 @@ public void testNonIntersectingSchemas() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 2L) - .put("index", 100.0D) - .put("addRowsIndexConstant", 103.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 2L) + .put("index", 100.0D) + .put("addRowsIndexConstant", 103.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : 100.0D) + .build() ) ) ); @@ -747,13 +748,13 @@ public void testValueAndEmptySchemas() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 2L) - .put("index", 100.0D) - .put("addRowsIndexConstant", 103.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 2L) + .put("index", 100.0D) + .put("addRowsIndexConstant", 103.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : 100.0D) + .build() ) ) ); @@ -865,19 +866,19 @@ public void testValueAndEmptySchemas() @Test public void testEmptySchemas() { + List> expectedTimeseriesResults = Arrays.asList( new Result<>( DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 1L) - .put("index", 0.0D) - .put("addRowsIndexConstant", 2.0D) - .put("uniques", 0.0D) - .put("maxIndex", 0.0D) - .put("minIndex", 0.0D) - .build() - ) + TestHelper.createExpectedMap( + "rows", 1L, + "index", NullHandling.useDefaultValuesForNull() ? 0.0D : null, + "addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 2.0D : null, + "uniques", 0.0D, + "maxIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : null, + "minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : null + )) ) ); @@ -885,14 +886,14 @@ public void testEmptySchemas() new Result<>( DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( - ImmutableMap.builder() - .put("rows", 0L) - .put("index", 0.0D) - .put("addRowsIndexConstant", 1.0D) - .put("uniques", 0.0D) - .put("maxIndex", Double.NEGATIVE_INFINITY) - .put("minIndex", Double.POSITIVE_INFINITY) - .build() + TestHelper.createExpectedMap( + "rows", 0L, + "index", NullHandling.useDefaultValuesForNull() ? 0.0D : null, + "addRowsIndexConstant", NullHandling.useDefaultValuesForNull() ? 1.0D : null, + "uniques", 0.0D, + "maxIndex", NullHandling.useDefaultValuesForNull() ? Double.NEGATIVE_INFINITY : null, + "minIndex", NullHandling.useDefaultValuesForNull() ? Double.POSITIVE_INFINITY : null + ) ) ) ); @@ -1186,13 +1187,13 @@ public void testDifferentMetrics() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 11L) - .put("index", 900.0D) - .put("addRowsIndexConstant", 912.0D) - .put("uniques", UNIQUES_1) - .put("maxIndex", 100.0D) - .put("minIndex", 0.0D) - .build() + .put("rows", 11L) + .put("index", 900.0D) + .put("addRowsIndexConstant", 912.0D) + .put("uniques", UNIQUES_1) + .put("maxIndex", 100.0D) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0D : 100.0D) + .build() ) ) ); diff --git a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java index 2c04eca92b7d..36e60487ff78 100644 --- a/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java +++ b/processing/src/test/java/io/druid/segment/SchemalessTestSimpleTest.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; -import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.Druids; import io.druid.query.QueryPlus; import io.druid.query.QueryRunner; @@ -42,9 +42,9 @@ import io.druid.query.aggregation.post.ArithmeticPostAggregator; import io.druid.query.aggregation.post.ConstantPostAggregator; import io.druid.query.aggregation.post.FieldAccessPostAggregator; -import io.druid.query.search.SearchResultValue; import io.druid.query.search.SearchHit; import io.druid.query.search.SearchQuery; +import io.druid.query.search.SearchResultValue; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import io.druid.query.timeboundary.TimeBoundaryQuery; @@ -56,6 +56,7 @@ import io.druid.query.topn.TopNQueryBuilder; import io.druid.query.topn.TopNResultValue; import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -146,13 +147,13 @@ public void testFullOnTimeseries() DateTimes.of("2011-01-12T00:00:00.000Z"), new TimeseriesResultValue( ImmutableMap.builder() - .put("rows", 11L) - .put("index", 900.0) - .put("addRowsIndexConstant", 912.0) - .put("uniques", 2.000977198748901D) - .put("maxIndex", 100.0) - .put("minIndex", 0.0) - .build() + .put("rows", 11L) + .put("index", 900.0) + .put("addRowsIndexConstant", 912.0) + .put("uniques", 2.000977198748901D) + .put("maxIndex", 100.0) + .put("minIndex", NullHandling.useDefaultValuesForNull() ? 0.0 : 100.0) + .build() ) ) ); @@ -195,36 +196,36 @@ public void testFullOnTopN() Arrays.asList( new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("market", "spot") - .put("rows", 4L) - .put("index", 400.0D) - .put("addRowsIndexConstant", 405.0D) - .put("uniques", 1.0002442201269182D) - .put("maxIndex", 100.0) - .put("minIndex", 100.0) - .build() + .put("market", "spot") + .put("rows", 4L) + .put("index", 400.0D) + .put("addRowsIndexConstant", 405.0D) + .put("uniques", 1.0002442201269182D) + .put("maxIndex", 100.0) + .put("minIndex", 100.0) + .build() ), new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("market", "") - .put("rows", 2L) - .put("index", 200.0D) - .put("addRowsIndexConstant", 203.0D) - .put("uniques", 0.0) - .put("maxIndex", 100.0D) - .put("minIndex", 100.0D) - .build() + .put("market", "") + .put("rows", 2L) + .put("index", 200.0D) + .put("addRowsIndexConstant", 203.0D) + .put("uniques", 0.0) + .put("maxIndex", 100.0D) + .put("minIndex", 100.0D) + .build() ), new DimensionAndMetricValueExtractor( ImmutableMap.builder() - .put("market", "total_market") - .put("rows", 2L) - .put("index", 200.0D) - .put("addRowsIndexConstant", 203.0D) - .put("uniques", 1.0002442201269182D) - .put("maxIndex", 100.0D) - .put("minIndex", 100.0D) - .build() + .put("market", "total_market") + .put("rows", 2L) + .put("index", 200.0D) + .put("addRowsIndexConstant", 203.0D) + .put("uniques", 1.0002442201269182D) + .put("maxIndex", 100.0D) + .put("minIndex", 100.0D) + .build() ) ) ) diff --git a/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java b/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java index 682d82d21e92..d44d0cc6cf43 100644 --- a/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java +++ b/processing/src/test/java/io/druid/segment/TestDoubleColumnSelector.java @@ -29,4 +29,10 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { // Don't care about runtime shape in tests } + + @Override + public boolean isNull() + { + return false; + } } diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index 976aa9fecfd4..4fbd2b7721c9 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -21,7 +21,9 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.jackson.DefaultObjectMapper; @@ -315,12 +317,16 @@ private static void assertRow(String msg, Row expected, Row actual) final Object actualValue = actualMap.get(key); if (expectedValue instanceof Float || expectedValue instanceof Double) { - Assert.assertEquals( - StringUtils.format("%s: key[%s]", msg, key), - ((Number) expectedValue).doubleValue(), - ((Number) actualValue).doubleValue(), - Math.abs(((Number) expectedValue).doubleValue() * 1e-6) - ); + if (expectedValue == null) { + Assert.assertNull(actualValue); + } else { + Assert.assertEquals( + StringUtils.format("%s: key[%s]", msg, key), + ((Number) expectedValue).doubleValue(), + ((Number) actualValue).doubleValue(), + Math.abs(((Number) expectedValue).doubleValue() * 1e-6) + ); + } } else { Assert.assertEquals( StringUtils.format("%s: key[%s]", msg, key), @@ -330,4 +336,16 @@ private static void assertRow(String msg, Row expected, Row actual) } } } + + + public static Map createExpectedMap(Object... vals) + { + Preconditions.checkArgument(vals.length % 2 == 0); + + Map theVals = Maps.newHashMap(); + for (int i = 0; i < vals.length; i += 2) { + theVals.put(vals[i].toString(), vals[i + 1]); + } + return theVals; + } } diff --git a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index 2671b5479770..ccf0ab066e83 100644 --- a/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Test; @@ -99,7 +100,7 @@ private void checkBasicAPIs(String[] strings, Indexed index, boolean all { Assert.assertEquals(strings.length, index.size()); for (int i = 0; i < strings.length; i++) { - Assert.assertEquals(strings[i], index.get(i)); + Assert.assertEquals("Index " + i, strings[i], index.get(i)); } if (allowReverseLookup) { @@ -136,4 +137,37 @@ private GenericIndexed serializeAndDeserialize(GenericIndexed in Assert.assertEquals(0, byteBuffer.remaining()); return deserialized; } + + @Test + public void testNullSerializationWithoutCache() throws Exception + { + final String[] strings = {null, "", "c", "d", "e", "f", "g", "h", "i", "k", "j", "l"}; + + Indexed deserialized = serializeAndDeserialize( + GenericIndexed.fromArray( + strings, GenericIndexed.STRING_STRATEGY + ) + ); + Assert.assertNull(deserialized.get(0)); + Assert.assertEquals(deserialized.get(1), NullHandling.emptyToNullIfNeeded("")); + } + + @Test + public void testNullSerialization() throws Exception + { + final String[] strings = {null, "", "c", "d", "e", "f", "g", "h", "i", "k", "j", "l"}; + + GenericIndexed unCached = serializeAndDeserialize( + GenericIndexed.fromArray( + strings, GenericIndexed.STRING_STRATEGY + )); + + Indexed deserialized = new CachingIndexed( + unCached, CachingIndexed.INITIAL_CACHE_CAPACITY); + + Assert.assertNull(deserialized.get(0)); + Assert.assertEquals(deserialized.get(1), NullHandling.emptyToNullIfNeeded("")); + } + + } diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java index 0e6c7bb318ef..b34d52e6e08d 100644 --- a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -88,6 +89,21 @@ public static void tearDown() throws Exception @Test public void testLexicographicMatchEverything() + { + final List filters = ImmutableList.of( + new BoundDimFilter("dim0", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim1", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim2", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), + new BoundDimFilter("dim3", null, "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) + ); + + for (BoundDimFilter filter : filters) { + assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + } + } + + @Test + public void testLexicographicMatchWithEmptyString() { final List filters = ImmutableList.of( new BoundDimFilter("dim0", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), @@ -95,9 +111,15 @@ public void testLexicographicMatchEverything() new BoundDimFilter("dim2", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC), new BoundDimFilter("dim3", "", "z", false, false, false, null, StringComparators.LEXICOGRAPHIC) ); - - for (BoundDimFilter filter : filters) { - assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + if (NullHandling.useDefaultValuesForNull()) { + for (BoundDimFilter filter : filters) { + assertFilterMatches(filter, ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + } + } else { + assertFilterMatches(filters.get(0), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + assertFilterMatches(filters.get(1), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")); + assertFilterMatches(filters.get(2), ImmutableList.of("0", "2", "3", "4", "6", "7")); + assertFilterMatches(filters.get(3), ImmutableList.of()); } } @@ -112,19 +134,49 @@ public void testLexicographicMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("1", "2", "5") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("1", "2", "5") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("2") + ); + } } @Test public void testLexicographicMatchMissingColumn() { - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + assertFilterMatches( + new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } assertFilterMatches( new BoundDimFilter("dim3", "", "", true, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of() @@ -133,18 +185,11 @@ public void testLexicographicMatchMissingColumn() new BoundDimFilter("dim3", "", "", false, true, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of() ); - assertFilterMatches( - new BoundDimFilter("dim3", "", null, false, true, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + assertFilterMatches( new BoundDimFilter("dim3", null, "", false, false, false, null, StringComparators.LEXICOGRAPHIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim3", null, "", false, true, false, null, StringComparators.LEXICOGRAPHIC), - ImmutableList.of() - ); } @@ -229,14 +274,25 @@ public void testAlphaNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, true, null, StringComparators.ALPHANUMERIC), + ImmutableList.of() + ); + } } @Test @@ -327,14 +383,26 @@ public void testNumericMatchNull() new BoundDimFilter("dim1", "", "", false, false, false, null, StringComparators.NUMERIC), ImmutableList.of("0") ); - assertFilterMatches( - new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim2", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter("dim3", "", "", false, false, false, null, StringComparators.NUMERIC), + ImmutableList.of() + ); + } + } @Test @@ -432,10 +500,17 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { return null; }"; ExtractionFn makeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches( - new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new BoundDimFilter("dim0", "", "", false, false, false, makeNullFn, StringComparators.LEXICOGRAPHIC), + ImmutableList.of() + ); + } assertFilterMatches( new BoundDimFilter("dim1", "super-ab", "super-abd", true, true, false, superFn, StringComparators.LEXICOGRAPHIC), @@ -452,10 +527,69 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim2", "super-null", "super-null", false, false, false, superFn, StringComparators.LEXICOGRAPHIC), - ImmutableList.of("1", "2", "5") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "2", "5") + ); + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.NUMERIC + ), + ImmutableList.of("1", "2", "5") + ); + } else { + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim2", "super-", "super-", false, false, false, superFn, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + assertFilterMatches( + new BoundDimFilter( + "dim2", + "super-null", + "super-null", + false, + false, + false, + superFn, + StringComparators.LEXICOGRAPHIC + ), + ImmutableList.of("1", "5") + ); + assertFilterMatches( + new BoundDimFilter("dim2", "super-", "super-", false, false, false, superFn, StringComparators.NUMERIC), + ImmutableList.of("2") + ); + } assertFilterMatches( new BoundDimFilter("dim3", "super-null", "super-null", false, false, false, superFn, StringComparators.LEXICOGRAPHIC), @@ -467,11 +601,6 @@ public void testMatchWithExtractionFn() ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") ); - assertFilterMatches( - new BoundDimFilter("dim2", "super-null", "super-null", false, false, false, superFn, StringComparators.NUMERIC), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( new BoundDimFilter("dim4", "super-null", "super-null", false, false, false, superFn, StringComparators.NUMERIC), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") diff --git a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java index 1b320c6ef8f2..99728789002d 100644 --- a/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ColumnComparisonFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -123,14 +124,28 @@ public void testMissingColumnNotSpecifiedInDimensionList() DefaultDimensionSpec.of("dim6"), DefaultDimensionSpec.of("dim7") )), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( - DefaultDimensionSpec.of("dim1"), - DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("0")); - assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( - DefaultDimensionSpec.of("dim2"), - DefaultDimensionSpec.of("dim6") - )), ImmutableList.of("1", "2", "6", "7", "8")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim1"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("0")); + + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim2"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1", "2", "6", "7", "8")); + } else { + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim1"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of()); + + assertFilterMatches(new ColumnComparisonDimFilter(ImmutableList.of( + DefaultDimensionSpec.of("dim2"), + DefaultDimensionSpec.of("dim6") + )), ImmutableList.of("1", "6", "7", "8")); + } + } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java index 08cc28dd6a79..302ec6db9678 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExpressionFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.FloatDimensionSchema; @@ -130,8 +131,13 @@ public void testOneMultiValuedStringColumn() { // Expressions currently treat multi-valued arrays as nulls. // This test is just documenting the current behavior, not necessarily saying it makes sense. - - assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("0", "1", "2", "4", "5", "6", "7", "8")); + } else { + assertFilterMatches(EDF("dim4 == ''"), ImmutableList.of("2")); + // AS per SQL standard null == null returns false. + assertFilterMatches(EDF("dim4 == null"), ImmutableList.of()); + } assertFilterMatches(EDF("dim4 == '1'"), ImmutableList.of()); assertFilterMatches(EDF("dim4 == '3'"), ImmutableList.of("3")); } @@ -188,12 +194,25 @@ public void testCompareColumns() @Test public void testMissingColumn() { - assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(EDF("missing == ''"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } else { + // AS per SQL standard null == null returns false. + assertFilterMatches(EDF("missing == null"), ImmutableList.of()); + } assertFilterMatches(EDF("missing == '1'"), ImmutableList.of()); assertFilterMatches(EDF("missing == 2"), ImmutableList.of()); - assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.useDefaultValuesForNull()) { + // missing equivaluent to 0 + assertFilterMatches(EDF("missing < '2'"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing < 2"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + } else { + // missing equivalent to null + assertFilterMatches(EDF("missing < '2'"), ImmutableList.of()); + assertFilterMatches(EDF("missing < 2"), ImmutableList.of()); + assertFilterMatches(EDF("missing < 2.0"), ImmutableList.of()); + } assertFilterMatches(EDF("missing > '2'"), ImmutableList.of()); assertFilterMatches(EDF("missing > 2"), ImmutableList.of()); assertFilterMatches(EDF("missing > 2.0"), ImmutableList.of()); diff --git a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java index 5bd793a571b8..0b742ec7c2c2 100644 --- a/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java +++ b/processing/src/test/java/io/druid/segment/filter/FilterPartitionTest.java @@ -24,6 +24,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -116,7 +117,9 @@ public Filter toFilter() if (extractionFn == null) { return new NoBitmapSelectorFilter(dimension, value); } else { + //CHECKSTYLE.OFF: Regexp final String valueOrNull = Strings.emptyToNull(value); + //CHECKSTYLE.ON: Regexp final DruidPredicateFactory predicateFactory = new DruidPredicateFactory() { @Override @@ -199,7 +202,11 @@ public static void tearDown() throws Exception @Test public void testSinglePreFilterWithNulls() { - assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + } else { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new SelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); @@ -212,7 +219,11 @@ public void testSinglePreFilterWithNulls() @Test public void testSinglePostFilterWithNulls() { - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", null, null), ImmutableList.of()); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "", null), ImmutableList.of("0")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); @@ -221,8 +232,11 @@ public void testSinglePostFilterWithNulls() assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "abc", null), ImmutableList.of("5", "8")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "ab", null), ImmutableList.of()); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN), ImmutableList.of("0")); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN), ImmutableList.of("0")); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-10", JS_EXTRACTION_FN), ImmutableList.of("1")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), ImmutableList.of("2")); assertFilterMatches(new NoBitmapSelectorDimFilter("dim1", "super-1", JS_EXTRACTION_FN), ImmutableList.of("3", "9")); @@ -234,13 +248,23 @@ public void testSinglePostFilterWithNulls() @Test public void testBasicPreAndPostFilterWithNulls() { - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim2", "a", null), - new NoBitmapSelectorDimFilter("dim1", null, null) - )), - ImmutableList.of("0") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "a", null), + new NoBitmapSelectorDimFilter("dim1", null, null) + )), + ImmutableList.of("0") + ); + } else { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "a", null), + new NoBitmapSelectorDimFilter("dim1", null, null) + )), + ImmutableList.of() + ); + } assertFilterMatches( new AndDimFilter(Arrays.asList( @@ -274,14 +298,51 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of() ); - - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("0") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("0") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("2") + ); + } else { + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("0") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim2", "super-a", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim1", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of() + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("2") + ); + assertFilterMatches( + new AndDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of() + ); + } assertFilterMatches( new AndDimFilter(Arrays.asList( @@ -291,14 +352,6 @@ public void testBasicPreAndPostFilterWithNulls() ImmutableList.of("1") ); - assertFilterMatches( - new AndDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("2") - ); - assertFilterMatches( new AndDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-1", JS_EXTRACTION_FN), @@ -327,13 +380,23 @@ public void testOrPostFilterWithNulls() ImmutableList.of("0", "3") ); - assertFilterMatches( - new OrDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "abc", null), - new NoBitmapSelectorDimFilter("dim2", null, null) - )), - ImmutableList.of("1", "2", "5", "8") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "abc", null), + new NoBitmapSelectorDimFilter("dim2", null, null) + )), + ImmutableList.of("1", "2", "5", "8") + ); + } else { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "abc", null), + new NoBitmapSelectorDimFilter("dim2", null, null) + )), + ImmutableList.of("1", "5", "8") + ); + } assertFilterMatches( new OrDimFilter(Arrays.asList( @@ -382,13 +445,32 @@ public void testOrPostFilterWithNulls() )), ImmutableList.of("0", "3") ); - assertFilterMatches( - new OrDimFilter(Arrays.asList( - new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), - new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) - )), - ImmutableList.of("1", "2", "5", "8") - ); + + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("1", "2", "5", "8") + ); + } else { + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-null", JS_EXTRACTION_FN) + )), + ImmutableList.of("1", "5", "8") + ); + assertFilterMatches( + new OrDimFilter(Arrays.asList( + new SelectorDimFilter("dim1", "super-abc", JS_EXTRACTION_FN), + new NoBitmapSelectorDimFilter("dim2", "super-", JS_EXTRACTION_FN) + )), + ImmutableList.of("2", "5", "8") + ); + } + assertFilterMatches( new OrDimFilter(Arrays.asList( new SelectorDimFilter("dim1", "super-2", JS_EXTRACTION_FN), @@ -432,7 +514,14 @@ public void testOrPostFilterWithNulls() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim3", "", null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + ); + } else { + assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "", null), ImmutableList.of()); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "a", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "b", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim3", "c", null), ImmutableList.of()); @@ -482,7 +571,17 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); - assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim4", "", null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7", "8", "9") + ); + } else { + assertFilterMatches( + new NoBitmapSelectorDimFilter("dim4", "", null), + ImmutableList.of() + ); + } assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "a", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "b", null), ImmutableList.of()); assertFilterMatches(new NoBitmapSelectorDimFilter("dim4", "c", null), ImmutableList.of()); diff --git a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java index d7e1a14293e0..ed1d61efe081 100644 --- a/processing/src/test/java/io/druid/segment/filter/InFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/InFilterTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -125,10 +126,17 @@ public void testSingleValueStringColumnWithNulls() ImmutableList.of("a") ); - assertFilterMatches( - toInFilter("dim1", null, "10", "abc"), - ImmutableList.of("a", "b", "f") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilter("dim1", null, "10", "abc"), + ImmutableList.of("a", "b", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim1", null, "10", "abc"), + ImmutableList.of("b", "f") + ); + } assertFilterMatches( toInFilter("dim1", "-1", "ab", "de"), @@ -139,28 +147,47 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches( - toInFilter("dim2", null), - ImmutableList.of("b", "c", "f") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilter("dim2", null), + ImmutableList.of("b", "c", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "a"), + ImmutableList.of("a", "b", "c", "d", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "b"), + ImmutableList.of("a", "b", "c", "f") + ); + assertFilterMatches( + toInFilter("dim2", ""), + ImmutableList.of("b", "c", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim2", null), + ImmutableList.of("b", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "a"), + ImmutableList.of("a", "b", "d", "f") + ); + assertFilterMatches( + toInFilter("dim2", null, "b"), + ImmutableList.of("a", "b", "f") + ); + assertFilterMatches( + toInFilter("dim2", ""), + ImmutableList.of("c") + ); + } assertFilterMatches( toInFilter("dim2", "", (String) null), ImmutableList.of("b", "c", "f") ); - assertFilterMatches( - toInFilter("dim2", null, "a"), - ImmutableList.of("a", "b", "c", "d", "f") - - ); - - assertFilterMatches( - toInFilter("dim2", null, "b"), - ImmutableList.of("a", "b", "c", "f") - - ); - assertFilterMatches( toInFilter("dim2", "c"), ImmutableList.of("e") @@ -180,10 +207,17 @@ public void testMissingColumn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - assertFilterMatches( - toInFilter("dim3", ""), - ImmutableList.of("a", "b", "c", "d", "e", "f") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilter("dim3", ""), + ImmutableList.of("a", "b", "c", "d", "e", "f") + ); + } else { + assertFilterMatches( + toInFilter("dim3", ""), + ImmutableList.of() + ); + } assertFilterMatches( toInFilter("dim3", null, "a"), @@ -215,20 +249,43 @@ public void testMatchWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'YES'; } else { return 'NO';} }"; ExtractionFn yesNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches( - toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), - ImmutableList.of("a", "b", "c", "d", "f") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), + ImmutableList.of("a", "b", "c", "d", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), + ImmutableList.of("a", "b", "e") + ); + assertFilterMatches( + toInFilterWithFn("dim2", yesNullFn, "YES"), + ImmutableList.of("b", "c", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", yesNullFn, "NO"), + ImmutableList.of("b", "c", "d", "e", "f") + ); + } else { + assertFilterMatches( + toInFilterWithFn("dim2", superFn, "super-null", "super-a", "super-b"), + ImmutableList.of("a", "b", "d", "f") + ); + assertFilterMatches( + toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), + ImmutableList.of("b", "e") + ); + assertFilterMatches( + toInFilterWithFn("dim2", yesNullFn, "YES"), + ImmutableList.of("b", "f") + ); + + assertFilterMatches( + toInFilterWithFn("dim1", yesNullFn, "NO"), + ImmutableList.of("a", "b", "c", "d", "e", "f") + ); + } - assertFilterMatches( - toInFilterWithFn("dim2", yesNullFn, "YES"), - ImmutableList.of("b", "c", "f") - ); - - assertFilterMatches( - toInFilterWithFn("dim1", superFn, "super-null", "super-10", "super-def"), - ImmutableList.of("a", "b", "e") - ); assertFilterMatches( toInFilterWithFn("dim3", yesNullFn, "NO"), @@ -240,10 +297,6 @@ public void testMatchWithExtractionFn() ImmutableList.of("a", "b", "c", "d", "e", "f") ); - assertFilterMatches( - toInFilterWithFn("dim1", yesNullFn, "NO"), - ImmutableList.of("b", "c", "d", "e", "f") - ); } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java index 760ae0f7ad5a..240cc92e43cd 100644 --- a/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/JavaScriptFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -109,7 +110,12 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of("0")); + } else { + assertFilterMatches(newJavaScriptDimFilter("dim1", jsNullFilter, null), ImmutableList.of()); + assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter(""), null), ImmutableList.of("0")); + } assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("10"), null), ImmutableList.of("1")); assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("2"), null), ImmutableList.of("2")); assertFilterMatches(newJavaScriptDimFilter("dim1", jsValueFilter("1"), null), ImmutableList.of("3")); @@ -122,7 +128,12 @@ public void testSingleValueStringColumnWithNulls() public void testMultiValueStringColumn() { // multi-val null...... - assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(newJavaScriptDimFilter("dim2", jsNullFilter, null), ImmutableList.of("1", "5")); + assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter(""), null), ImmutableList.of("2")); + } assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("a"), null), ImmutableList.of("0", "3")); assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("b"), null), ImmutableList.of("0")); assertFilterMatches(newJavaScriptDimFilter("dim2", jsValueFilter("c"), null), ImmutableList.of("4")); diff --git a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java index b9c127a81bc0..fb989db93fb8 100644 --- a/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/LikeFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -156,10 +157,17 @@ public void testMatchEmptyString() @Test public void testMatchEmptyStringWithExtractionFn() { - assertFilterMatches( - new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new LikeDimFilter("dim1", "", null, new SubstringDimExtractionFn(100, 1)), + ImmutableList.of() + ); + } } @Test diff --git a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java index 732cbbcb8f78..98cd861d51f5 100644 --- a/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RegexFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -99,7 +100,11 @@ public void testSingleValueStringColumnWithoutNulls() public void testSingleValueStringColumnWithNulls() { // RegexFilter always returns false for null row values. - assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new RegexDimFilter("dim1", ".*", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } assertFilterMatches(new RegexDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new RegexDimFilter("dim1", "2", null), ImmutableList.of("2")); assertFilterMatches(new RegexDimFilter("dim1", "1", null), ImmutableList.of("1", "3")); @@ -111,7 +116,11 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "3", "4")); + } else { + assertFilterMatches(new RegexDimFilter("dim2", ".*", null), ImmutableList.of("0", "2", "3", "4")); + } assertFilterMatches(new RegexDimFilter("dim2", "a", null), ImmutableList.of("0", "3")); assertFilterMatches(new RegexDimFilter("dim2", "b", null), ImmutableList.of("0")); assertFilterMatches(new RegexDimFilter("dim2", "c", null), ImmutableList.of("4")); @@ -141,11 +150,16 @@ public void testRegexWithExtractionFn() { String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - - assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of("0")); + assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(new RegexDimFilter("dim1", ".*ANYMORE", changeNullFn), ImmutableList.of()); + assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "5")); + } assertFilterMatches(new RegexDimFilter("dim1", "ab.*", changeNullFn), ImmutableList.of("4", "5")); - assertFilterMatches(new RegexDimFilter("dim2", ".*ANYMORE", changeNullFn), ImmutableList.of("1", "2", "5")); + assertFilterMatches(new RegexDimFilter("dim2", "a.*", changeNullFn), ImmutableList.of("0", "3")); assertFilterMatches(new RegexDimFilter("dim3", ".*ANYMORE", changeNullFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java index b1cf8293457f..16a885a8d0ce 100644 --- a/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SearchQueryFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -105,8 +106,18 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - // SearchQueryFilter always returns false for null row values. - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue(""), null), ImmutableList.of("1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + // SearchQueryFilter always returns false for null row values. + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue(""), null), + ImmutableList.of("1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue(""), null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("10"), null), ImmutableList.of("1")); assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("2"), null), ImmutableList.of("2")); assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("1"), null), ImmutableList.of("1", "3")); @@ -118,7 +129,14 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue(""), null), ImmutableList.of("0", "3", "4")); + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue(""), null), + ImmutableList.of("0", "2", "3", "4") + ); + } assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("a"), null), ImmutableList.of("0", "3")); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("b"), null), ImmutableList.of("0")); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("c"), null), ImmutableList.of("4")); @@ -151,10 +169,31 @@ public void testSearchQueryWithExtractionFn() String nullJsFn = "function(str) { if (str === null) { return 'NOT_NULL_ANYMORE'; } else { return str;} }"; ExtractionFn changeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0")); - assertFilterMatches(new SearchQueryDimFilter("dim1", specForValue("ab"), changeNullFn), ImmutableList.of("4", "5")); - - assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("1", "2", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("0") + ); + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("1", "2", "5") + ); + + } else { + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of() + ); + assertFilterMatches( + new SearchQueryDimFilter("dim2", specForValue("ANYMORE"), changeNullFn), + ImmutableList.of("1", "5") + ); + } + + assertFilterMatches( + new SearchQueryDimFilter("dim1", specForValue("ab"), changeNullFn), + ImmutableList.of("4", "5") + ); assertFilterMatches(new SearchQueryDimFilter("dim2", specForValue("a"), changeNullFn), ImmutableList.of("0", "3")); assertFilterMatches(new SearchQueryDimFilter("dim3", specForValue("ANYMORE"), changeNullFn), ImmutableList.of("0", "1", "2", "3", "4", "5")); diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java index c3981d4f7e26..05a3e104d2d2 100644 --- a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -113,8 +114,13 @@ public void testSingleValueStringColumnWithoutNulls() @Test public void testSingleValueStringColumnWithNulls() { - assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); - assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of("0")); + assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + } else { + assertFilterMatches(new SelectorDimFilter("dim1", null, null), ImmutableList.of()); + assertFilterMatches(new SelectorDimFilter("dim1", "", null), ImmutableList.of("0")); + } assertFilterMatches(new SelectorDimFilter("dim1", "10", null), ImmutableList.of("1")); assertFilterMatches(new SelectorDimFilter("dim1", "2", null), ImmutableList.of("2")); assertFilterMatches(new SelectorDimFilter("dim1", "1", null), ImmutableList.of("3")); @@ -126,8 +132,13 @@ public void testSingleValueStringColumnWithNulls() @Test public void testMultiValueStringColumn() { - assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); - assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "2", "5")); + assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("1", "2", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim2", null, null), ImmutableList.of("1", "5")); + assertFilterMatches(new SelectorDimFilter("dim2", "", null), ImmutableList.of("2")); + } assertFilterMatches(new SelectorDimFilter("dim2", "a", null), ImmutableList.of("0", "3")); assertFilterMatches(new SelectorDimFilter("dim2", "b", null), ImmutableList.of("0")); assertFilterMatches(new SelectorDimFilter("dim2", "c", null), ImmutableList.of("4")); @@ -138,7 +149,11 @@ public void testMultiValueStringColumn() public void testMissingColumnSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim3", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim3", "", null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim3", "a", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim3", "b", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim3", "c", null), ImmutableList.of()); @@ -148,7 +163,11 @@ public void testMissingColumnSpecifiedInDimensionList() public void testMissingColumnNotSpecifiedInDimensionList() { assertFilterMatches(new SelectorDimFilter("dim4", null, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); - assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(new SelectorDimFilter("dim4", "", null), ImmutableList.of()); + } assertFilterMatches(new SelectorDimFilter("dim4", "a", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim4", "b", null), ImmutableList.of()); assertFilterMatches(new SelectorDimFilter("dim4", "c", null), ImmutableList.of()); @@ -200,7 +219,23 @@ public void testSelectorWithLookupExtractionFn() ); LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false); LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true); - assertFilterMatches(new SelectorDimFilter("dim0", null, lookupFn3), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + // Nulls and empty strings are considered equivalent + assertFilterMatches( + new SelectorDimFilter("dim0", null, lookupFn3), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new SelectorDimFilter("dim0", null, lookupFn3), + ImmutableList.of("0", "2", "3", "4", "5") + ); + assertFilterMatches( + new SelectorDimFilter("dim0", "", lookupFn3), + ImmutableList.of("1") + ); + } + final Map stringMap4 = ImmutableMap.of( "9", "4" @@ -228,6 +263,7 @@ public void testSelectorWithLookupExtractionFn() SelectorDimFilter optFilter5 = new SelectorDimFilter("dim0", "5", lookupFn5); SelectorDimFilter optFilter6 = new SelectorDimFilter("dim0", "5", lookupFn6); + InDimFilter optFilter2Optimized = new InDimFilter("dim0", Arrays.asList("2", "5"), null); SelectorDimFilter optFilter4Optimized = new SelectorDimFilter("dim0", "5", null); SelectorDimFilter optFilter6Optimized = new SelectorDimFilter("dim0", "5", null); @@ -238,10 +274,15 @@ public void testSelectorWithLookupExtractionFn() Assert.assertTrue(optFilter4Optimized.equals(optFilter4.optimize())); Assert.assertTrue(optFilter5.equals(optFilter5.optimize())); Assert.assertTrue(optFilter6Optimized.equals(optFilter6.optimize())); - assertFilterMatches(optFilter1, ImmutableList.of("0", "1", "2", "5")); + assertFilterMatches(optFilter2, ImmutableList.of("2", "5")); - assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + // Null and Empty strings are same + assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5")); + } else { + assertFilterMatches(optFilter3, ImmutableList.of("0", "2", "3", "4", "5")); + } assertFilterMatches(optFilter4, ImmutableList.of("5")); assertFilterMatches(optFilter5, ImmutableList.of()); assertFilterMatches(optFilter6, ImmutableList.of("5")); @@ -250,6 +291,20 @@ public void testSelectorWithLookupExtractionFn() // remove these when ExtractionDimFilter is removed. assertFilterMatches(new ExtractionDimFilter("dim1", "UNKNOWN", lookupFn, null), ImmutableList.of("0", "1", "2", "5")); assertFilterMatches(new ExtractionDimFilter("dim0", "5", lookupFn2, null), ImmutableList.of("2", "5")); - assertFilterMatches(new ExtractionDimFilter("dim0", null, lookupFn3, null), ImmutableList.of("0", "1", "2", "3", "4", "5")); + if (NullHandling.useDefaultValuesForNull()) { + assertFilterMatches( + new ExtractionDimFilter("dim0", null, lookupFn3, null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + } else { + assertFilterMatches( + new ExtractionDimFilter("dim0", null, lookupFn3, null), + ImmutableList.of("0", "2", "3", "4", "5") + ); + assertFilterMatches( + new ExtractionDimFilter("dim0", "", lookupFn3, null), + ImmutableList.of("1") + ); + } } } diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 73feff0787b3..73759dbd0cb5 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; @@ -537,9 +538,13 @@ public void testCursoringAndSnapshot() throws Exception // no null id, so should get empty dims array Assert.assertEquals(0, rowD.size()); IndexedInts rowE = dimSelector3E.getRow(); - Assert.assertEquals(1, rowE.size()); - // the null id - Assert.assertEquals(0, rowE.get(0)); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(1, rowE.size()); + // the null id + Assert.assertEquals(0, rowE.get(0)); + } else { + Assert.assertEquals(0, rowE.size()); + } cursor.advance(); rowNumInCursor++; } diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java index ee728cb51605..3ba15dad45e1 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.druid.collections.StupidPool; +import io.druid.common.config.NullHandling; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.Row; import io.druid.data.input.impl.DimensionSchema; @@ -224,10 +225,23 @@ public void testNullDimensionTransform() throws IndexSizeExceededException Row row = index.iterator().next(); - Assert.assertEquals(Arrays.asList(new String[]{"", "", "A"}), row.getRaw("string")); - Assert.assertEquals(Arrays.asList(new String[]{"", "", String.valueOf(Float.POSITIVE_INFINITY)}), row.getRaw("float")); - Assert.assertEquals(Arrays.asList(new String[]{"", "", String.valueOf(Long.MIN_VALUE)}), row.getRaw("long")); - Assert.assertEquals(0.0, row.getMetric("double").doubleValue(), 0.0); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(Arrays.asList(null, null, "A"), row.getRaw("string")); + Assert.assertEquals( + Arrays.asList(null, null, String.valueOf(Float.POSITIVE_INFINITY)), + row.getRaw("float") + ); + Assert.assertEquals(Arrays.asList(null, null, String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); + } else { + Assert.assertEquals(Arrays.asList(null, "", "A"), row.getRaw("string")); + Assert.assertEquals( + Arrays.asList(null, "", String.valueOf(Float.POSITIVE_INFINITY)), + row.getRaw("float") + ); + Assert.assertEquals(Arrays.asList(null, "", String.valueOf(Long.MIN_VALUE)), row.getRaw("long")); + } + Assert.assertEquals(null, row.getRaw("double")); + } @Test diff --git a/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java new file mode 100644 index 000000000000..da65317a26c6 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/DoubleGenericColumnPartSerdeTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteOrder; + +public class DoubleGenericColumnPartSerdeTest +{ + private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); + + @Test + public void testSerdeV2() throws IOException + { + DoubleGenericColumnPartSerdeV2 object = DoubleGenericColumnPartSerdeV2.getDoubleGenericColumnPartSerde( + ByteOrder.BIG_ENDIAN, + new RoaringBitmapSerdeFactory(true) + ); + ColumnPartSerde columnPartSerde = jsonMapper.readValue( + jsonMapper.writeValueAsString(object), + ColumnPartSerde.class + ); + Assert.assertTrue(columnPartSerde instanceof DoubleGenericColumnPartSerdeV2); + Assert.assertEquals( + object.getBitmapSerdeFactory(), + ((DoubleGenericColumnPartSerdeV2) columnPartSerde).getBitmapSerdeFactory() + ); + + } +} diff --git a/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java new file mode 100644 index 000000000000..4abad1d4688a --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/FloatGenericColumnPartSerdeTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteOrder; + +public class FloatGenericColumnPartSerdeTest +{ + private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); + + @Test + public void testSerdeV2() throws IOException + { + FloatGenericColumnPartSerdeV2 object = FloatGenericColumnPartSerdeV2.createDeserializer( + ByteOrder.BIG_ENDIAN, + new RoaringBitmapSerdeFactory(true) + ); + ColumnPartSerde columnPartSerde = jsonMapper.readValue( + jsonMapper.writeValueAsString(object), + ColumnPartSerde.class + ); + Assert.assertTrue(columnPartSerde instanceof FloatGenericColumnPartSerdeV2); + Assert.assertEquals( + object.getBitmapSerdeFactory(), + ((FloatGenericColumnPartSerdeV2) columnPartSerde).getBitmapSerdeFactory() + ); + + } +} diff --git a/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java b/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java new file mode 100644 index 000000000000..4712aa99b241 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/serde/LongGenericColumnPartSerdeTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.serde; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.TestHelper; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteOrder; + +public class LongGenericColumnPartSerdeTest +{ + private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); + + @Test + public void testSerdeV2() throws IOException + { + LongGenericColumnPartSerdeV2 object = LongGenericColumnPartSerdeV2.createDeserializer( + ByteOrder.BIG_ENDIAN, + new RoaringBitmapSerdeFactory(true) + ); + ColumnPartSerde columnPartSerde = jsonMapper.readValue( + jsonMapper.writeValueAsString(object), + ColumnPartSerde.class + ); + Assert.assertTrue(columnPartSerde instanceof LongGenericColumnPartSerdeV2); + Assert.assertEquals( + object.getBitmapSerdeFactory(), + ((LongGenericColumnPartSerdeV2) columnPartSerde).getBitmapSerdeFactory() + ); + + } +} diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java index 8e8e8c1349e4..bf9ca098c0e8 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionColumnValueSelectorTest.java @@ -82,7 +82,6 @@ public void testSupplierFromObjectSelectorNumber() objectSelectorFromSupplier(settableSupplier, Number.class) ); - Assert.assertNotNull(supplier); Assert.assertEquals(null, supplier.get()); diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index d132e1a6673f..bc31c22605ff 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.Row; @@ -113,7 +114,12 @@ public void testObjectSelector() Assert.assertEquals(null, selector.getObject()); CURRENT_ROW.set(ROW1); - Assert.assertEquals(4.0d, selector.getObject()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(4.0d, selector.getObject()); + } else { + // y is null for row1 + Assert.assertEquals(null, selector.getObject()); + } CURRENT_ROW.set(ROW2); Assert.assertEquals(5.1d, selector.getObject()); @@ -131,7 +137,12 @@ public void testLongSelector() Assert.assertEquals(0L, selector.getLong()); CURRENT_ROW.set(ROW1); - Assert.assertEquals(4L, selector.getLong()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(4L, selector.getLong()); + } else { + // y is null for row1 + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW2); Assert.assertEquals(5L, selector.getLong()); @@ -149,7 +160,12 @@ public void testLongSelectorUsingStringFunction() Assert.assertEquals(0L, selector.getLong()); CURRENT_ROW.set(ROW1); - Assert.assertEquals(4L, selector.getLong()); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(4L, selector.getLong()); + } else { + // y is null for row1 + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW2); Assert.assertEquals(0L, selector.getLong()); @@ -167,7 +183,12 @@ public void testFloatSelector() Assert.assertEquals(0.0f, selector.getFloat(), 0.0f); CURRENT_ROW.set(ROW1); - Assert.assertEquals(4.0f, selector.getFloat(), 0.0f); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(4.0f, selector.getFloat(), 0.0f); + } else { + // y is null for row1 + Assert.assertTrue(selector.isNull()); + } CURRENT_ROW.set(ROW2); Assert.assertEquals(5.1f, selector.getFloat(), 0.0f); @@ -195,10 +216,18 @@ public void testDimensionSelector() Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); CURRENT_ROW.set(ROW1); - Assert.assertEquals(false, nullMatcher.matches()); - Assert.assertEquals(false, fiveMatcher.matches()); - Assert.assertEquals(true, nonNullMatcher.matches()); - Assert.assertEquals("4.0", selector.lookupName(selector.getRow().get(0))); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("4.0", selector.lookupName(selector.getRow().get(0))); + } else { + // y is null in row1 + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + } CURRENT_ROW.set(ROW2); Assert.assertEquals(false, nullMatcher.matches()); @@ -229,7 +258,10 @@ public void testDimensionSelectorUsingStringFunction() CURRENT_ROW.set(ROW1); Assert.assertEquals(1, selector.getRow().size()); - Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0))); + Assert.assertEquals( + NullHandling.useDefaultValuesForNull() ? "4" : null, + selector.lookupName(selector.getRow().get(0)) + ); CURRENT_ROW.set(ROW2); Assert.assertEquals(1, selector.getRow().size()); @@ -259,10 +291,18 @@ public void testDimensionSelectorWithExtraction() Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); CURRENT_ROW.set(ROW1); - Assert.assertEquals(false, nullMatcher.matches()); - Assert.assertEquals(false, fiveMatcher.matches()); - Assert.assertEquals(true, nonNullMatcher.matches()); - Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0))); + if (NullHandling.useDefaultValuesForNull()) { + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0))); + } else { + // y is null in row1 + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + } CURRENT_ROW.set(ROW2); Assert.assertEquals(false, nullMatcher.matches()); diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index 943b50258ca6..53e4464fe137 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -379,6 +379,12 @@ public Class classOfObject() { return String.class; } + + @Override + public boolean isNull() + { + return selector.isNull(); + } }; return dimensionSpec.decorate(dimensionSelector); @@ -397,6 +403,12 @@ public long getLong() { return theLong; } + + @Override + public boolean isNull() + { + return false; + } }; } diff --git a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java index 97448a55cb10..953d5ff199a6 100644 --- a/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java +++ b/server/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java @@ -31,6 +31,7 @@ import io.druid.query.lookup.LookupExtractor; import io.druid.query.lookup.LookupReferencesManager; import io.druid.segment.DimensionSelector; +import io.druid.common.config.NullHandling; import io.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -77,7 +78,7 @@ public LookupDimensionSpec( { this.retainMissingValue = retainMissingValue; this.optimize = optimize == null ? true : optimize; - this.replaceMissingValueWith = Strings.emptyToNull(replaceMissingValueWith); + this.replaceMissingValueWith = NullHandling.emptyToNullIfNeeded(replaceMissingValueWith); this.dimension = Preconditions.checkNotNull(dimension, "dimension can not be Null"); this.outputName = Preconditions.checkNotNull(outputName, "outputName can not be Null"); this.lookupReferencesManager = lookupReferencesManager; @@ -167,14 +168,15 @@ public boolean mustDecorate() public byte[] getCacheKey() { byte[] dimensionBytes = StringUtils.toUtf8(dimension); + //CHECKSTYLE.OFF: Regexp byte[] dimExtractionFnBytes = Strings.isNullOrEmpty(name) ? getLookup().getCacheKey() : StringUtils.toUtf8(name); byte[] outputNameBytes = StringUtils.toUtf8(outputName); byte[] replaceWithBytes = StringUtils.toUtf8(Strings.nullToEmpty(replaceMissingValueWith)); + //CHECKSTYLE.ON: Regexp - - return ByteBuffer.allocate(6 + return ByteBuffer.allocate(7 + dimensionBytes.length + outputNameBytes.length + dimExtractionFnBytes.length @@ -189,6 +191,7 @@ public byte[] getCacheKey() .put(replaceWithBytes) .put(DimFilterUtils.STRING_SEPARATOR) .put(retainMissingValue ? (byte) 1 : (byte) 0) + .put(replaceMissingValueWith == null ? (byte) 0 : (byte) 1) .array(); } diff --git a/server/src/main/java/io/druid/query/expression/LookupExprMacro.java b/server/src/main/java/io/druid/query/expression/LookupExprMacro.java index 77dc88c184bd..e24ad3ff8c71 100644 --- a/server/src/main/java/io/druid/query/expression/LookupExprMacro.java +++ b/server/src/main/java/io/druid/query/expression/LookupExprMacro.java @@ -19,8 +19,8 @@ package io.druid.query.expression; -import com.google.common.base.Strings; import com.google.inject.Inject; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.IAE; import io.druid.math.expr.Expr; import io.druid.math.expr.ExprEval; @@ -77,7 +77,7 @@ class LookupExpr implements Expr @Override public ExprEval eval(final ObjectBinding bindings) { - return ExprEval.of(extractionFn.apply(Strings.emptyToNull(arg.eval(bindings).asString()))); + return ExprEval.of(extractionFn.apply(NullHandling.emptyToNullIfNeeded(arg.eval(bindings).asString()))); } @Override diff --git a/server/src/main/java/io/druid/query/lookup/LookupModule.java b/server/src/main/java/io/druid/query/lookup/LookupModule.java index a1d387cf7225..e855aa320022 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupModule.java +++ b/server/src/main/java/io/druid/query/lookup/LookupModule.java @@ -261,7 +261,9 @@ public String getLookupTier() ); final String lookupTier = lookupTierIsDatasource ? dataSourceTaskIdHolder.getDataSource() : this.lookupTier; return Preconditions.checkNotNull( + //CHECKSTYLE.OFF: Regexp lookupTier == null ? DEFAULT_TIER : Strings.emptyToNull(lookupTier), + //CHECKSTYLE.ON: Regexp "Cannot have empty lookup tier from %s", lookupTierIsDatasource ? "bound value" : LookupModule.PROPERTY_BASE ); diff --git a/server/src/main/java/io/druid/server/QueryLifecycle.java b/server/src/main/java/io/druid/server/QueryLifecycle.java index 1f25f370e279..959367778f4e 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycle.java +++ b/server/src/main/java/io/druid/server/QueryLifecycle.java @@ -293,7 +293,9 @@ public void emitLogsAndMetrics( queryMetricsFactory, toolChest, queryPlus.getQuery(), + //CHECKSTYLE.OFF: Regexp Strings.nullToEmpty(remoteAddress) + //CHECKSTYLE.ON: Regexp ); queryMetrics.success(success); queryMetrics.reportQueryTime(queryTimeNs); @@ -325,7 +327,9 @@ public void emitLogsAndMetrics( requestLogger.log( new RequestLogLine( DateTimes.utc(startMs), + //CHECKSTYLE.OFF: Regexp Strings.nullToEmpty(remoteAddress), + //CHECKSTYLE.ON: Regexp queryPlus.getQuery(), new QueryStats(statsMap) ) diff --git a/server/src/main/java/io/druid/server/emitter/EmitterModule.java b/server/src/main/java/io/druid/server/emitter/EmitterModule.java index a0e08225ee1a..fbb614b6728f 100644 --- a/server/src/main/java/io/druid/server/emitter/EmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/EmitterModule.java @@ -88,7 +88,9 @@ public void configure(Binder binder) String version = getClass().getPackage().getImplementationVersion(); extraServiceDimensions .addBinding("version") + //CHECKSTYLE.OFF: Regexp .toInstance(Strings.nullToEmpty(version)); // Version is null during `mvn test`. + //CHECKSTYLE.ON: Regexp } @Provides diff --git a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java index 35288b6bc3f9..71d55e2d07fa 100644 --- a/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java +++ b/server/src/main/java/io/druid/server/listener/announcer/ListeningAnnouncerConfig.java @@ -93,7 +93,9 @@ public String getAnnouncementPath(String listenerName) { return ZKPaths.makePath( getListenersPath(), Preconditions.checkNotNull( + //CHECKSTYLE.OFF: Regexp Strings.emptyToNull(listenerName), "Listener name cannot be null" + //CHECKSTYLE.ON: Regexp ) ); } diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index 7e1ef423f959..ddd2108f4b25 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -160,7 +160,9 @@ public Object[] parametersForTestApply() public void testApply(DimensionSpec dimensionSpec, Map map) { for (Map.Entry entry : map.entrySet()) { + //CHECKSTYLE.OFF: Regexp Assert.assertEquals(Strings.emptyToNull(entry.getValue()), dimensionSpec.getExtractionFn().apply(entry.getKey())); + //CHECKSTYLE.ON: Regexp } } diff --git a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java index 5dc3135ed965..8a1f9b632745 100644 --- a/server/src/test/java/io/druid/query/expression/ExprMacroTest.java +++ b/server/src/test/java/io/druid/query/expression/ExprMacroTest.java @@ -23,6 +23,7 @@ import io.druid.java.util.common.DateTimes; import io.druid.math.expr.Expr; import io.druid.math.expr.Parser; +import io.druid.common.config.NullHandling; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -43,6 +44,9 @@ public class ExprMacroTest .build() ); + public static String emptyStringResult = NullHandling.useDefaultValuesForNull() ? null : ""; + + @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -84,8 +88,8 @@ public void testRegexpExtract() public void testTimestampCeil() { assertExpr("timestamp_ceil(t, 'P1M')", DateTimes.of("2000-03-01").getMillis()); - assertExpr("timestamp_ceil(t, 'P1D','','America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis()); - assertExpr("timestamp_ceil(t, 'P1D','',CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D', null,'America/Los_Angeles')", DateTimes.of("2000-02-03T08").getMillis()); + assertExpr("timestamp_ceil(t, 'P1D', null,CityOfAngels)", DateTimes.of("2000-02-03T08").getMillis()); assertExpr("timestamp_ceil(t, 'P1D','1970-01-01T01','Etc/UTC')", DateTimes.of("2000-02-04T01").getMillis()); } @@ -93,8 +97,8 @@ public void testTimestampCeil() public void testTimestampFloor() { assertExpr("timestamp_floor(t, 'P1M')", DateTimes.of("2000-02-01").getMillis()); - assertExpr("timestamp_floor(t, 'P1D','','America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis()); - assertExpr("timestamp_floor(t, 'P1D','',CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D', null,'America/Los_Angeles')", DateTimes.of("2000-02-02T08").getMillis()); + assertExpr("timestamp_floor(t, 'P1D', null,CityOfAngels)", DateTimes.of("2000-02-02T08").getMillis()); assertExpr("timestamp_floor(t, 'P1D','1970-01-01T01','Etc/UTC')", DateTimes.of("2000-02-03T01").getMillis()); } @@ -122,7 +126,7 @@ public void testTimestampParse() assertExpr("timestamp_parse(tstr)", DateTimes.of("2000-02-03T04:05:06").getMillis()); assertExpr("timestamp_parse(tstr_sql)", DateTimes.of("2000-02-03T04:05:06").getMillis()); assertExpr( - "timestamp_parse(tstr_sql,'','America/Los_Angeles')", + "timestamp_parse(tstr_sql,null,'America/Los_Angeles')", DateTimes.of("2000-02-03T04:05:06-08:00").getMillis() ); assertExpr("timestamp_parse('2000-02-03')", DateTimes.of("2000-02-03").getMillis()); @@ -148,36 +152,36 @@ public void testTimestampFormat() @Test public void testTrim() { - assertExpr("trim('')", null); + assertExpr("trim('')", emptyStringResult); assertExpr("trim(concat(' ',x,' '))", "foo"); assertExpr("trim(spacey)", "hey there"); assertExpr("trim(spacey, '')", " hey there "); assertExpr("trim(spacey, 'he ')", "y ther"); - assertExpr("trim(spacey, spacey)", null); + assertExpr("trim(spacey, spacey)", emptyStringResult); assertExpr("trim(spacey, substring(spacey, 0, 4))", "y ther"); } @Test public void testLTrim() { - assertExpr("ltrim('')", null); + assertExpr("ltrim('')", emptyStringResult); assertExpr("ltrim(concat(' ',x,' '))", "foo "); assertExpr("ltrim(spacey)", "hey there "); assertExpr("ltrim(spacey, '')", " hey there "); assertExpr("ltrim(spacey, 'he ')", "y there "); - assertExpr("ltrim(spacey, spacey)", null); + assertExpr("ltrim(spacey, spacey)", emptyStringResult); assertExpr("ltrim(spacey, substring(spacey, 0, 4))", "y there "); } @Test public void testRTrim() { - assertExpr("rtrim('')", null); + assertExpr("rtrim('')", emptyStringResult); assertExpr("rtrim(concat(' ',x,' '))", " foo"); assertExpr("rtrim(spacey)", " hey there"); assertExpr("rtrim(spacey, '')", " hey there "); assertExpr("rtrim(spacey, 'he ')", " hey ther"); - assertExpr("rtrim(spacey, spacey)", null); + assertExpr("rtrim(spacey, spacey)", emptyStringResult); assertExpr("rtrim(spacey, substring(spacey, 0, 4))", " hey ther"); } diff --git a/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java b/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java index f1332e21ec5f..571aeb5f169a 100644 --- a/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java +++ b/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java @@ -38,7 +38,8 @@ public class RegisteredLookupExtractionFnTest { private static Map MAP = ImmutableMap.of( "foo", "bar", - "bat", "baz" + "bat", "baz", + "", "empty" ); private static final LookupExtractor LOOKUP_EXTRACTOR = new MapLookupExtractor(MAP, true); private static final String LOOKUP_NAME = "some lookup"; diff --git a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java index 59f145555e85..17ad17256a8e 100644 --- a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java +++ b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java @@ -831,7 +831,7 @@ public List getDimension(String dimension) @Override public Number getMetric(String metric) { - return 0; + return 0F; } @Override diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java index cfddca4aa90e..45c2bf02e8f4 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java @@ -607,7 +607,7 @@ public List getDimension(String dimension) @Override public Number getMetric(String metric) { - return 0; + return 0D; } @Override @@ -655,7 +655,7 @@ public List getDimension(String dimension) @Override public Number getMetric(String metric) { - return 0; + return 0F; } @Override diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java index dce3e7a89780..5b3f6cc4dd83 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/SinkTest.java @@ -116,7 +116,7 @@ public List getDimension(String dimension) @Override public Number getMetric(String metric) { - return 0; + return 0F; } @Override @@ -170,7 +170,7 @@ public List getDimension(String dimension) @Override public Number getMetric(String metric) { - return 0; + return 0F; } @Override diff --git a/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index 27e789c4c83a..45de020e8d37 100644 --- a/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/server/src/test/java/io/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -33,6 +33,7 @@ import org.junit.Assert; import org.junit.Test; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; @@ -241,6 +242,7 @@ public List getDimension(String s) } @Override + @Nullable public Object getRaw(String s) { return null; @@ -249,7 +251,7 @@ public Object getRaw(String s) @Override public Number getMetric(String metric) { - return 0; + return 0F; } @Override diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index 794e9181cbae..3d82fbf8d3ce 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; -import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; @@ -41,6 +40,7 @@ import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.common.config.NullHandling; import io.druid.guice.DruidProcessingModule; import io.druid.guice.QueryRunnerFactoryModule; import io.druid.guice.QueryableModule; @@ -364,18 +364,20 @@ public Object apply(final OutputStream out) jg.writeFieldName(columnName); jg.writeStartObject(); for (int i = 0; i < bitmapIndex.getCardinality(); i++) { - jg.writeFieldName(Strings.nullToEmpty(bitmapIndex.getValue(i))); - final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); - if (decompressBitmaps) { - jg.writeStartArray(); - final IntIterator iterator = bitmap.iterator(); - while (iterator.hasNext()) { - final int rowNum = iterator.next(); - jg.writeNumber(rowNum); + String val = NullHandling.nullToEmptyIfNeeded(bitmapIndex.getValue(i)); + if (val != null) { + final ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); + if (decompressBitmaps) { + jg.writeStartArray(); + final IntIterator iterator = bitmap.iterator(); + while (iterator.hasNext()) { + final int rowNum = iterator.next(); + jg.writeNumber(rowNum); + } + jg.writeEndArray(); + } else { + jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap)); } - jg.writeEndArray(); - } else { - jg.writeBinary(bitmapSerdeFactory.getObjectStrategy().toBytes(bitmap)); } } jg.writeEndObject(); @@ -552,5 +554,11 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) { inspector.visit("delegate", delegate); } + + @Override + public boolean isNull() + { + return delegate.isNull(); + } } } diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index b779fdd03a0c..69c3915a3b41 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -90,7 +90,7 @@ public static String stringLiteral(final String s) public static String nullLiteral() { - return "''"; + return "null"; } public static String functionCall(final String functionName, final List args) diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java index 3cff45e36546..b87d052574f4 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/Expressions.java @@ -33,14 +33,15 @@ import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.TimeFormatExtractionFn; import io.druid.query.filter.AndDimFilter; -import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; import io.druid.query.filter.ExpressionDimFilter; import io.druid.query.filter.LikeDimFilter; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.SelectorDimFilter; import io.druid.query.ordering.StringComparator; import io.druid.query.ordering.StringComparators; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.segment.column.ValueType; import io.druid.sql.calcite.filtration.BoundRefKey; @@ -308,13 +309,10 @@ private static DimFilter toSimpleLeafFilter( return null; } - final BoundDimFilter equalFilter = Bounds.equalTo( - new BoundRefKey( - druidExpression.getSimpleExtraction().getColumn(), - druidExpression.getSimpleExtraction().getExtractionFn(), - StringComparators.LEXICOGRAPHIC - ), - "" + final DimFilter equalFilter = new SelectorDimFilter( + druidExpression.getSimpleExtraction().getColumn(), + NullHandling.defaultStringValue(), + druidExpression.getSimpleExtraction().getExtractionFn() ); return kind == SqlKind.IS_NOT_NULL ? new NotDimFilter(equalFilter) : equalFilter; diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java new file mode 100644 index 000000000000..5465e311b752 --- /dev/null +++ b/sql/src/main/java/io/druid/sql/calcite/expression/UnaryFunctionOperatorConversion.java @@ -0,0 +1,66 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.sql.calcite.expression; + +import com.google.common.collect.Iterables; +import io.druid.java.util.common.StringUtils; +import io.druid.sql.calcite.planner.PlannerContext; +import io.druid.sql.calcite.table.RowSignature; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperator; + +public class UnaryFunctionOperatorConversion implements SqlOperatorConversion +{ + private final SqlOperator operator; + private final String druidOperator; + + public UnaryFunctionOperatorConversion(final SqlOperator operator, final String druidOperator) + { + this.operator = operator; + this.druidOperator = druidOperator; + } + + @Override + public SqlOperator calciteOperator() + { + return operator; + } + + @Override + public DruidExpression toDruidExpression( + final PlannerContext plannerContext, + final RowSignature rowSignature, + final RexNode rexNode + ) + { + return OperatorConversions.convertCall( + plannerContext, + rowSignature, + rexNode, + operands -> DruidExpression.fromExpression( + StringUtils.format( + "%s(%s)", + druidOperator, + Iterables.getOnlyElement(operands).getExpression() + ) + ) + ); + } +} diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java index 216d7bcbcb14..5d4bd4b9fbdc 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/CeilOperatorConversion.java @@ -19,7 +19,6 @@ package io.druid.sql.calcite.expression.builtin; -import com.google.common.collect.ImmutableList; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.sql.calcite.expression.DruidExpression; @@ -35,6 +34,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import java.util.Arrays; import java.util.stream.Collectors; public class CeilOperatorConversion implements SqlOperatorConversion @@ -80,7 +80,7 @@ public DruidExpression toDruidExpression( // So there is no simple extraction for this operator. return DruidExpression.fromFunctionCall( "timestamp_ceil", - ImmutableList.of( + Arrays.asList( druidExpression.getExpression(), DruidExpression.stringLiteral(granularity.getPeriod().toString()), DruidExpression.numberLiteral( diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java index ac566a686059..ed6a94376547 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/Calcites.java @@ -19,6 +19,7 @@ package io.druid.sql.calcite.planner; +import com.google.common.base.Preconditions; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Chars; import io.druid.java.util.common.DateTimes; @@ -104,6 +105,7 @@ public static SchemaPlus createRootSchema(final Schema druidSchema, final Author public static String escapeStringLiteral(final String s) { + Preconditions.checkNotNull(s); if (s == null) { return "''"; } else { diff --git a/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java index b2ea2e183d76..59fab5b55201 100644 --- a/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/io/druid/sql/calcite/planner/DruidOperatorTable.java @@ -37,6 +37,7 @@ import io.druid.sql.calcite.expression.BinaryOperatorConversion; import io.druid.sql.calcite.expression.DirectOperatorConversion; import io.druid.sql.calcite.expression.SqlOperatorConversion; +import io.druid.sql.calcite.expression.UnaryFunctionOperatorConversion; import io.druid.sql.calcite.expression.UnaryPrefixOperatorConversion; import io.druid.sql.calcite.expression.UnarySuffixOperatorConversion; import io.druid.sql.calcite.expression.builtin.BTrimOperatorConversion; @@ -117,8 +118,8 @@ public class DruidOperatorTable implements SqlOperatorTable .add(new DirectOperatorConversion(SqlStdOperatorTable.UPPER, "upper")) .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.NOT, "!")) .add(new UnaryPrefixOperatorConversion(SqlStdOperatorTable.UNARY_MINUS, "-")) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NULL, "== ''")) - .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "!= ''")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NULL, "isnull")) + .add(new UnaryFunctionOperatorConversion(SqlStdOperatorTable.IS_NOT_NULL, "notnull")) .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_FALSE, "<= 0")) // Matches Evals.asBoolean .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_NOT_TRUE, "<= 0")) // Matches Evals.asBoolean .add(new UnarySuffixOperatorConversion(SqlStdOperatorTable.IS_TRUE, "> 0")) // Matches Evals.asBoolean diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java index 32ff1206f4d6..4fc4212623bb 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidRel.java @@ -87,7 +87,6 @@ public boolean isValidDruidQuery() * Convert this DruidRel to a DruidQuery for purposes of explaining. This must be an inexpensive operation. For * example, DruidSemiJoin will use a dummy dataSource in order to complete this method, rather than executing * the right-hand side query. - * * This method may not return null. * * @return query diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java index 5d6abfc1f01a..f4b89e6d2c8d 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/DruidSemiJoin.java @@ -28,6 +28,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.ResourceLimitExceededException; +import io.druid.segment.DimensionHandlerUtils; import io.druid.sql.calcite.planner.PlannerContext; import org.apache.calcite.interpreter.BindableConvention; import org.apache.calcite.plan.RelOptCluster; @@ -294,7 +295,11 @@ public List accumulate(final List theConditions, final Object[ for (int i : rightKeys) { final Object value = row[i]; - final String stringValue = value != null ? String.valueOf(value) : ""; + if (value == null) { + // NULLS are not supposed to match NULLs in a join. So ignore them. + continue; + } + final String stringValue = DimensionHandlerUtils.convertObjectToString(value); values.add(stringValue); if (values.size() > maxSemiJoinRowsInMemory) { throw new ResourceLimitExceededException( @@ -308,16 +313,18 @@ public List accumulate(final List theConditions, final Object[ for (int i = 0; i < values.size(); i++) { final String value = values.get(i); - subConditions.add( - getCluster().getRexBuilder().makeCall( - SqlStdOperatorTable.EQUALS, - leftExpressions.get(i), - getCluster().getRexBuilder().makeLiteral(value) - ) - ); + // NULLS are not supposed to match NULLs in a join. So ignore them. + if (value != null) { + subConditions.add( + getCluster().getRexBuilder().makeCall( + SqlStdOperatorTable.EQUALS, + leftExpressions.get(i), + getCluster().getRexBuilder().makeLiteral(value) + ) + ); + } + theConditions.add(makeAnd(subConditions)); } - - theConditions.add(makeAnd(subConditions)); } return theConditions; } diff --git a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java index ed006a8b95ab..46554fd4cea5 100644 --- a/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java +++ b/sql/src/main/java/io/druid/sql/calcite/rel/QueryMaker.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -46,6 +45,7 @@ import io.druid.query.topn.TopNQuery; import io.druid.query.topn.TopNResultValue; import io.druid.segment.DimensionHandlerUtils; +import io.druid.common.config.NullHandling; import io.druid.segment.column.Column; import io.druid.server.QueryLifecycleFactory; import io.druid.server.security.AuthenticationResult; @@ -398,7 +398,7 @@ private Object coerce(final Object value, final SqlTypeName sqlType) if (SqlTypeName.CHAR_TYPES.contains(sqlType)) { if (value == null || value instanceof String) { - coercedValue = Strings.nullToEmpty((String) value); + coercedValue = NullHandling.nullToEmptyIfNeeded((String) value); } else if (value instanceof NlsString) { coercedValue = ((NlsString) value).getValue(); } else if (value instanceof Number) { diff --git a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java index d229549d366e..b5653cdebee3 100644 --- a/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/io/druid/sql/avatica/DruidStatementTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprMacroTable; +import io.druid.common.config.NullHandling; import io.druid.server.security.AllowAllAuthenticator; import io.druid.server.security.NoopEscalator; import io.druid.server.security.AuthConfig; @@ -142,6 +143,7 @@ public void testSelectAllInFirstFrame() throws Exception Meta.Frame.create( 0, true, + NullHandling.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f}, @@ -149,6 +151,14 @@ public void testSelectAllInFirstFrame() throws Exception new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", "", 6.0f} + ) : + Lists.newArrayList( + new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, + new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", null, 2.0f}, + new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, + new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, + new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, + new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", null, 6.0f} ) ), frame @@ -162,16 +172,21 @@ public void testSelectSplitOverTwoFrames() throws Exception final String sql = "SELECT __time, cnt, dim1, dim2, m1 FROM druid.foo"; final DruidStatement statement = new DruidStatement("", 0, null, () -> { }).prepare(plannerFactory, sql, -1, AllowAllAuthenticator.ALLOW_ALL_RESULT); - + // First frame, ask for 2 rows. Meta.Frame frame = statement.execute().nextFrame(DruidStatement.START_OFFSET, 2); Assert.assertEquals( Meta.Frame.create( 0, false, + NullHandling.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", "", 2.0f} + ) : + Lists.newArrayList( + new Object[]{DateTimes.of("2000-01-01").getMillis(), 1L, "", "a", 1.0f}, + new Object[]{DateTimes.of("2000-01-02").getMillis(), 1L, "10.1", null, 2.0f} ) ), frame @@ -184,11 +199,18 @@ public void testSelectSplitOverTwoFrames() throws Exception Meta.Frame.create( 2, true, + NullHandling.useDefaultValuesForNull() ? Lists.newArrayList( new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", "", 6.0f} + ) : + Lists.newArrayList( + new Object[]{DateTimes.of("2000-01-03").getMillis(), 1L, "2", "", 3.0f}, + new Object[]{DateTimes.of("2001-01-01").getMillis(), 1L, "1", "a", 4.0f}, + new Object[]{DateTimes.of("2001-01-02").getMillis(), 1L, "def", "abc", 5.0f}, + new Object[]{DateTimes.of("2001-01-03").getMillis(), 1L, "abc", null, 6.0f} ) ), frame diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 163760cc8d89..9ded67f3077a 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import io.druid.common.config.NullHandling; import io.druid.hll.HLLCV1; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -87,6 +88,7 @@ import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.ForbiddenException; +import io.druid.sql.calcite.expression.DruidExpression; import io.druid.sql.calcite.filtration.Filtration; import io.druid.sql.calcite.planner.Calcites; import io.druid.sql.calcite.planner.DruidOperatorTable; @@ -476,6 +478,7 @@ public void testExplainInformationSchemaColumns() throws Exception @Test public void testSelectStar() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo", ImmutableList.of( @@ -489,11 +492,11 @@ public void testSelectStar() throws Exception ), ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2f, 2.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -501,6 +504,7 @@ public void testSelectStar() throws Exception @Test public void testSelectStarOnForbiddenTable() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; assertQueryIsForbidden( "SELECT * FROM druid.forbiddenDatasource", CalciteTests.REGULAR_USER_AUTH_RESULT @@ -520,7 +524,7 @@ public void testSelectStarOnForbiddenTable() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, 0.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-01"), 1L, "forbidden", "abcd", 9999.0f, nullValue, HLLCV1.class.getName()} ) ); } @@ -562,6 +566,8 @@ public void testExplainSelectStar() throws Exception @Test public void testSelectStarWithLimit() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT * FROM druid.foo LIMIT 2", ImmutableList.of( @@ -576,7 +582,7 @@ public void testSelectStarWithLimit() throws Exception ), ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1.0f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2.0f, 2.0, HLLCV1.class.getName()} + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2.0f, 2.0, HLLCV1.class.getName()} ) ); } @@ -584,6 +590,7 @@ public void testSelectStarWithLimit() throws Exception @Test public void testSelectWithProjection() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT SUBSTRING(dim2, 1, 1) FROM druid.foo LIMIT 2", ImmutableList.of( @@ -601,7 +608,7 @@ public void testSelectWithProjection() throws Exception ), ImmutableList.of( new Object[]{"a"}, - new Object[]{""} + new Object[]{nullValue} ) ); } @@ -609,6 +616,8 @@ public void testSelectWithProjection() throws Exception @Test public void testSelectStarWithLimitTimeDescending() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT * FROM druid.foo ORDER BY __time DESC LIMIT 2", ImmutableList.of( @@ -624,7 +633,7 @@ public void testSelectStarWithLimitTimeDescending() throws Exception .build() ), ImmutableList.of( - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6d, HLLCV1.class.getName()}, + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6d, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5d, HLLCV1.class.getName()} ) ); @@ -633,6 +642,7 @@ public void testSelectStarWithLimitTimeDescending() throws Exception @Test public void testSelectStarWithoutLimitTimeAscending() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT * FROM druid.foo ORDER BY __time", ImmutableList.of( @@ -665,11 +675,11 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception ), ImmutableList.of( new Object[]{T("2000-01-01"), 1L, "", "a", 1f, 1.0, HLLCV1.class.getName()}, - new Object[]{T("2000-01-02"), 1L, "10.1", "", 2f, 2.0, HLLCV1.class.getName()}, + new Object[]{T("2000-01-02"), 1L, "10.1", nullValue, 2f, 2.0, HLLCV1.class.getName()}, new Object[]{T("2000-01-03"), 1L, "2", "", 3f, 3.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-01"), 1L, "1", "a", 4f, 4.0, HLLCV1.class.getName()}, new Object[]{T("2001-01-02"), 1L, "def", "abc", 5f, 5.0, HLLCV1.class.getName()}, - new Object[]{T("2001-01-03"), 1L, "abc", "", 6f, 6.0, HLLCV1.class.getName()} + new Object[]{T("2001-01-03"), 1L, "abc", nullValue, 6f, 6.0, HLLCV1.class.getName()} ) ); } @@ -677,6 +687,7 @@ public void testSelectStarWithoutLimitTimeAscending() throws Exception @Test public void testSelectSingleColumnTwice() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim2 x, dim2 y FROM druid.foo LIMIT 2", ImmutableList.of( @@ -691,7 +702,7 @@ public void testSelectSingleColumnTwice() throws Exception ), ImmutableList.of( new Object[]{"a", "a"}, - new Object[]{"", ""} + new Object[]{nullValue, nullValue} ) ); } @@ -797,9 +808,12 @@ public void testSelfJoinWithFallback() throws Exception @Test public void testExplainSelfJoinWithFallback() throws Exception { + String emptyStringEq = NullHandling.useDefaultValuesForNull() ? null : "\"\""; final String explanation = "BindableJoin(condition=[=($0, $2)], joinType=[inner])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING}])\n" + + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":" + + emptyStringEq + + ",\"extractionFn\":null}},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"dim1\",\"dim2\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING, dim2:STRING}])\n"; testQuery( @@ -1146,9 +1160,14 @@ public void testHavingOnApproximateCountDistinct() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L}, new Object[]{"a", 2L} + ) : + ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"a", 2L} ) ); } @@ -1198,9 +1217,14 @@ public void testHavingOnExactCountDistinct() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L}, new Object[]{"a", 2L} + ) : + ImmutableList.of( + new Object[]{null, 2L}, + new Object[]{"a", 2L} ) ); } @@ -1316,6 +1340,7 @@ public void testHavingOnRatio() throws Exception @Test public void testGroupByWithSelectProjections() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1335,10 +1360,10 @@ public void testGroupByWithSelectProjections() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", ""}, + new Object[]{"", nullValue}, + new Object[]{"1", nullValue}, new Object[]{"10.1", "0.1"}, - new Object[]{"2", ""}, + new Object[]{"2", nullValue}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"} ) @@ -1348,6 +1373,7 @@ public void testGroupByWithSelectProjections() throws Exception @Test public void testGroupByWithSelectAndOrderByProjections() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " dim1," @@ -1387,9 +1413,9 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception new Object[]{"10.1", "0.1"}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"}, - new Object[]{"1", ""}, - new Object[]{"2", ""}, - new Object[]{"", ""} + new Object[]{"1", nullValue}, + new Object[]{"2", nullValue}, + new Object[]{"", nullValue} ) ); } @@ -1397,6 +1423,8 @@ public void testGroupByWithSelectAndOrderByProjections() throws Exception @Test public void testTopNWithSelectProjections() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT\n" + " dim1," @@ -1419,10 +1447,10 @@ public void testTopNWithSelectProjections() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, - new Object[]{"1", ""}, + new Object[]{"", nullValue}, + new Object[]{"1", nullValue}, new Object[]{"10.1", "0.1"}, - new Object[]{"2", ""}, + new Object[]{"2", nullValue}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"} ) @@ -1432,6 +1460,8 @@ public void testTopNWithSelectProjections() throws Exception @Test public void testTopNWithSelectAndOrderByProjections() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; + testQuery( "SELECT\n" + " dim1," @@ -1459,9 +1489,9 @@ public void testTopNWithSelectAndOrderByProjections() throws Exception new Object[]{"10.1", "0.1"}, new Object[]{"abc", "bc"}, new Object[]{"def", "ef"}, - new Object[]{"1", ""}, - new Object[]{"2", ""}, - new Object[]{"", ""} + new Object[]{"1", nullValue}, + new Object[]{"2", nullValue}, + new Object[]{"", nullValue} ) ); } @@ -1550,6 +1580,7 @@ public void testPruneDeadAggregatorsThroughHaving() throws Exception @Test public void testGroupByCaseWhen() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT\n" + " CASE EXTRACT(DAY FROM __time)\n" @@ -1580,7 +1611,7 @@ public void testGroupByCaseWhen() throws Exception + "'match-cnt'," + "(timestamp_extract(\"__time\",'DAY','UTC') == 0)," + "'zero '," - + "'')", + + DruidExpression.nullLiteral() + ")", ValueType.STRING ) ) @@ -1590,7 +1621,7 @@ public void testGroupByCaseWhen() throws Exception .build() ), ImmutableList.of( - new Object[]{"", 2L}, + new Object[]{nullValue, 2L}, new Object[]{"match-cnt", 1L}, new Object[]{"match-m1 ", 3L} ) @@ -1598,37 +1629,87 @@ public void testGroupByCaseWhen() throws Exception } @Test - public void testNullEmptyStringEquality() throws Exception + public void testIsNullString() throws Exception { - // Doesn't conform to the SQL standard, but it's how we do it. - // This example is used in the sql.md doc. + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') IS NULL", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,isnull(\"dim2\"))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + NullHandling.useDefaultValuesForNull() ? + // Matches everything but "abc" + new Object[]{5L} : + // match only null values + new Object[]{4L} + ) + ); + } + + @Test + public void testEmptyStringEquality() throws Exception + { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') = ''", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandling.useDefaultValuesForNull() ? "1" : "0") + + ",(\"dim2\" == ''))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + NullHandling.useDefaultValuesForNull() ? + // Matches everything but "abc" + new Object[]{5L} : + // match only empty string + new Object[]{1L} + ) + ); + } + + @Test + public void testNullStringEquality() throws Exception + { + testQuery( + "SELECT COUNT(*)\n" + + "FROM druid.foo\n" + + "WHERE NULLIF(dim2, 'a') = null", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a')," + + (NullHandling.useDefaultValuesForNull() ? "1" : "0") + + ",(\"dim2\" == null))")) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + NullHandling.useDefaultValuesForNull() ? + // Matches everything but "abc" + ImmutableList.of(new Object[]{5L}) : + // null is not eqaual to null or any other value + ImmutableList.of() + ); - final ImmutableList wheres = ImmutableList.of( - "NULLIF(dim2, 'a') = ''", - "NULLIF(dim2, 'a') IS NULL" - ); - - for (String where : wheres) { - testQuery( - "SELECT COUNT(*)\n" - + "FROM druid.foo\n" - + "WHERE " + where, - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Filtration.eternity())) - .granularity(Granularities.ALL) - .filters(EXPRESSION_FILTER("case_searched((\"dim2\" == 'a'),1,(\"dim2\" == ''))")) - .aggregators(AGGS(new CountAggregatorFactory("a0"))) - .context(TIMESERIES_CONTEXT_DEFAULT) - .build() - ), - ImmutableList.of( - // Matches everything but "abc" - new Object[]{5L} - ) - ); - } } @Test @@ -1647,7 +1728,7 @@ public void testCoalesceColumns() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "case_searched((\"dim2\" != ''),\"dim2\",\"dim1\")", + "case_searched(notnull(\"dim2\"),\"dim2\",\"dim1\")", ValueType.STRING ) ) @@ -1656,11 +1737,18 @@ public void testCoalesceColumns() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, new Object[]{"a", 2L}, new Object[]{"abc", 2L} + ) : + ImmutableList.of( + new Object[]{"", 1L}, + new Object[]{"10.1", 1L}, + new Object[]{"a", 2L}, + new Object[]{"abc", 2L} ) ); } @@ -1684,7 +1772,7 @@ public void testColumnIsNull() throws Exception .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{NullHandling.useDefaultValuesForNull() ? 3L : 2L} ) ); } @@ -1902,14 +1990,18 @@ public void testCountNullableColumn() throws Exception .aggregators(AGGS( new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), - NOT(SELECTOR("dim2", "", null)) + NOT(SELECTOR("dim2", null, null)) ) )) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L} + ) : + ImmutableList.of( + new Object[]{4L} ) ); } @@ -1928,7 +2020,9 @@ public void testCountNullableExpression() throws Exception new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), EXPRESSION_FILTER( - "(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes','') != '')" + "notnull(case_searched((\"dim2\" == 'abc'),'yes',(\"dim2\" == 'def'),'yes'," + + DruidExpression.nullLiteral() + + "))" ) ) )) @@ -2220,7 +2314,7 @@ public void testFilterOnStringAsNumber() throws Exception public void testSimpleAggregations() throws Exception { testQuery( - "SELECT COUNT(*), COUNT(cnt), COUNT(dim1), AVG(cnt), SUM(cnt), SUM(cnt) + MIN(cnt) + MAX(cnt) FROM druid.foo", + "SELECT COUNT(*), COUNT(cnt), COUNT(dim1), AVG(cnt), SUM(cnt), SUM(cnt) + MIN(cnt) + MAX(cnt), COUNT(dim2) FROM druid.foo", ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -2231,13 +2325,17 @@ public void testSimpleAggregations() throws Exception new CountAggregatorFactory("a0"), new FilteredAggregatorFactory( new CountAggregatorFactory("a1"), - NOT(SELECTOR("dim1", "", null)) + NOT(SELECTOR("dim1", null, null)) ), new LongSumAggregatorFactory("a2:sum", "cnt"), new CountAggregatorFactory("a2:count"), new LongSumAggregatorFactory("a3", "cnt"), new LongMinAggregatorFactory("a4", "cnt"), - new LongMaxAggregatorFactory("a5", "cnt") + new LongMaxAggregatorFactory("a5", "cnt"), + new FilteredAggregatorFactory( + new CountAggregatorFactory("a6"), + NOT(SELECTOR("dim2", null, null)) + ) ) ) .postAggregators( @@ -2254,8 +2352,12 @@ public void testSimpleAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{6L, 6L, 5L, 1L, 6L, 8L, 3L} + ) : ImmutableList.of( - new Object[]{6L, 6L, 5L, 1L, 6L, 8L} + new Object[]{6L, 6L, 6L, 1L, 6L, 8L, 4L} ) ); } @@ -2428,7 +2530,7 @@ public void testFilteredAggregations() throws Exception new FilteredAggregatorFactory( new CountAggregatorFactory("a3"), AND( - NOT(SELECTOR("dim2", "", null)), + NOT(SELECTOR("dim2", null, null)), NOT(SELECTOR("dim1", "1", null)) ) ), @@ -2473,8 +2575,12 @@ public void testFilteredAggregations() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{1L, 5L, 1L, 2L, 5L, 5L, 2L, 1L, 5L, 1L, 5L} + ) : + ImmutableList.of( + new Object[]{1L, 5L, 1L, 3L, 5L, 5L, 2L, 1L, 5L, 1L, 5L} ) ); } @@ -2542,8 +2648,12 @@ public void testFilteredAggregationWithNotIn() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{5L, 2L} + ) : + ImmutableList.of( + new Object[]{5L, 3L} ) ); } @@ -3377,7 +3487,7 @@ public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() thr .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) .setVirtualColumns( - EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"cnt\",'P1Y','','UTC')", ValueType.LONG) + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"cnt\",'P1Y',null,'UTC')", ValueType.LONG) ) .setDimFilter( BOUND( @@ -3482,7 +3592,14 @@ public void testSelectDistinctWithLimit() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} + ) : ImmutableList.of( + new Object[]{null}, new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} @@ -3506,10 +3623,17 @@ public void testSelectDistinctWithSortAsOuterQuery() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} + ) : + ImmutableList.of( + new Object[]{null}, + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} ) ); } @@ -3530,7 +3654,14 @@ public void testSelectDistinctWithSortAsOuterQuery2() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{""}, + new Object[]{"a"}, + new Object[]{"abc"} + ) : ImmutableList.of( + new Object[]{null}, new Object[]{""}, new Object[]{"a"}, new Object[]{"abc"} @@ -3566,10 +3697,17 @@ public void testSelectDistinctWithSortAsOuterQuery4() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{""}, new Object[]{"abc"}, new Object[]{"a"} + ) : + ImmutableList.of( + new Object[]{null}, + new Object[]{"abc"}, + new Object[]{"a"}, + new Object[]{""} ) ); } @@ -3684,14 +3822,14 @@ public void testExactCountDistinct() throws Exception .setAggregatorSpecs(AGGS( new FilteredAggregatorFactory( new CountAggregatorFactory("a0"), - NOT(SELECTOR("d0", "", null)) + NOT(SELECTOR("d0", null, null)) ) )) .setContext(QUERY_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{2L} + new Object[]{NullHandling.useDefaultValuesForNull() ? 2L : 3L} ) ); } @@ -3763,16 +3901,23 @@ public void testExactCountDistinctWithGroupingAndOtherAggregators() throws Excep new LongSumAggregatorFactory("_a0", "a0"), new FilteredAggregatorFactory( new CountAggregatorFactory("_a1"), - NOT(SELECTOR("d0", "", null)) + NOT(SELECTOR("d0", null, null)) ) )) .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 3L, 3L}, new Object[]{"a", 2L, 1L}, new Object[]{"abc", 1L, 1L} + ) : + ImmutableList.of( + new Object[]{null, 2L, 2L}, + new Object[]{"", 1L, 1L}, + new Object[]{"a", 2L, 2L}, + new Object[]{"abc", 1L, 1L} ) ); } @@ -3844,8 +3989,12 @@ public void testApproxCountDistinct() throws Exception .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L, 2L, 2L, 2L, 6L} + ) : + ImmutableList.of( + new Object[]{6L, 3L, 2L, 1L, 1L, 6L} ) ); } @@ -3890,7 +4039,7 @@ public void testNestedGroupBy() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "_d0:v", - "timestamp_floor(\"a0\",'PT1H','','UTC')", + "timestamp_floor(\"a0\",'PT1H',null,'UTC')", ValueType.LONG ) ) @@ -3958,8 +4107,12 @@ public void testDoubleNestedGroupBy() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 4L} ) ); } @@ -4024,8 +4177,12 @@ public void testExactCountDistinctUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{6L, 3L} + ) : + ImmutableList.of( + new Object[]{6L, 4L} ) ); } @@ -4033,6 +4190,9 @@ public void testExactCountDistinctUsingSubquery() throws Exception @Test public void testTopNFilterJoin() throws Exception { + DimFilter filter = NullHandling.useDefaultValuesForNull() ? + IN("dim2", Arrays.asList(null, "a"), null) + : SELECTOR("dim2", "a", null); // Filters on top N values of some dimension by using an inner join. testQuery( "SELECT t1.dim1, SUM(t1.cnt)\n" @@ -4063,7 +4223,7 @@ public void testTopNFilterJoin() throws Exception .setDataSource(CalciteTests.DATASOURCE1) .setInterval(QSS(Filtration.eternity())) .setGranularity(Granularities.ALL) - .setDimFilter(IN("dim2", ImmutableList.of("", "a"), null)) + .setDimFilter(filter) .setDimensions(DIMS(new DefaultDimensionSpec("dim1", "d0"))) .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) .setLimitSpec( @@ -4081,12 +4241,17 @@ public void testTopNFilterJoin() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", 1L}, new Object[]{"1", 1L}, new Object[]{"10.1", 1L}, new Object[]{"2", 1L}, new Object[]{"abc", 1L} + ) : + ImmutableList.of( + new Object[]{"", 1L}, + new Object[]{"1", 1L} ) ); } @@ -4267,7 +4432,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception final String explanation = "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"skipEmptyBuckets\":true,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"}}], signature=[{a0:LONG}])\n" + " DruidSemiJoin(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], leftExpressions=[[SUBSTRING($3, 1, 1)]], rightKeys=[[0]])\n" - + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{d0:STRING}])\n"; + + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{d0:STRING}])\n"; testQuery( "EXPLAIN PLAN FOR SELECT COUNT(*)\n" @@ -4275,7 +4440,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception + " SELECT DISTINCT dim2\n" + " FROM druid.foo\n" + " WHERE SUBSTRING(dim2, 1, 1) IN (\n" - + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 <> ''\n" + + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n" + " )\n" + ")", ImmutableList.of(), @@ -4316,8 +4481,51 @@ public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Except .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 2L} + ) : + ImmutableList.of( + new Object[]{5L, 3L} + ) + ); + + testQuery( + "SELECT\n" + + " SUM(cnt),\n" + + " COUNT(*)\n" + + "FROM (SELECT dim2, SUM(cnt) AS cnt FROM druid.foo GROUP BY dim2)\n" + + "WHERE dim2 IS NOT NULL", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource( + new QueryDataSource( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(QSS(Filtration.eternity())) + .setDimFilter(NOT(SELECTOR("dim2", null, null))) + .setGranularity(Granularities.ALL) + .setDimensions(DIMS(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ) + ) + .setInterval(QSS(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setAggregatorSpecs(AGGS( + new LongSumAggregatorFactory("_a0", "a0"), + new CountAggregatorFactory("_a1") + )) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + NullHandling.useDefaultValuesForNull() ? + ImmutableList.of( + new Object[]{3L, 2L} + ) : + ImmutableList.of( + new Object[]{4L, 3L} ) ); } @@ -4356,8 +4564,12 @@ public void testExactCountDistinctUsingSubqueryWithWhereToOuterFilter() throws E .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{3L, 1L} + ) : + ImmutableList.of( + new Object[]{2L, 1L} ) ); } @@ -4443,10 +4655,15 @@ public void testHistogramUsingSubquery() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L}, new Object[]{"3", 1L} + ) : + ImmutableList.of( + new Object[]{"1", 2L}, + new Object[]{"2", 2L} ) ); } @@ -4493,9 +4710,14 @@ public void testHistogramUsingSubqueryWithSort() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"1", 1L}, new Object[]{"2", 1L} + ) : + ImmutableList.of( + new Object[]{"1", 2L}, + new Object[]{"2", 2L} ) ); } @@ -4649,6 +4871,7 @@ public void testSillyQuarters() throws Exception @Test public void testRegexpExtract() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT DISTINCT\n" + " REGEXP_EXTRACT(dim1, '^.'),\n" @@ -4685,7 +4908,7 @@ public void testRegexpExtract() throws Exception .build() ), ImmutableList.of( - new Object[]{"", ""}, + new Object[]{nullValue, nullValue}, new Object[]{"1", "1"}, new Object[]{"2", "2"}, new Object[]{"a", "a"}, @@ -4697,6 +4920,7 @@ public void testRegexpExtract() throws Exception @Test public void testGroupBySortPushDown() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim2, dim1, SUM(cnt) FROM druid.foo GROUP BY dim2, dim1 ORDER BY dim1 LIMIT 4", ImmutableList.of( @@ -4729,7 +4953,7 @@ public void testGroupBySortPushDown() throws Exception ImmutableList.of( new Object[]{"a", "", 1L}, new Object[]{"a", "1", 1L}, - new Object[]{"", "10.1", 1L}, + new Object[]{nullValue, "10.1", 1L}, new Object[]{"", "2", 1L} ) ); @@ -4738,6 +4962,7 @@ public void testGroupBySortPushDown() throws Exception @Test public void testGroupByLimitPushDownWithHavingOnLong() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; testQuery( "SELECT dim1, dim2, SUM(cnt) AS thecnt " + "FROM druid.foo " @@ -4773,11 +4998,18 @@ public void testGroupByLimitPushDownWithHavingOnLong() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"10.1", "", 1L}, new Object[]{"2", "", 1L}, new Object[]{"abc", "", 1L}, new Object[]{"", "a", 1L} + ) : + ImmutableList.of( + new Object[]{"10.1", null, 1L}, + new Object[]{"abc", null, 1L}, + new Object[]{"2", "", 1L}, + new Object[]{"", "a", 1L} ) ); } @@ -5157,7 +5389,7 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(\"__time\",'P1Y','','UTC')", + "timestamp_floor(\"__time\",'P1Y',null,'UTC')", ValueType.LONG ) ) @@ -5197,12 +5429,21 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{T("2000"), "", 2L}, new Object[]{T("2000"), "a", 1L}, new Object[]{T("2001"), "", 1L}, new Object[]{T("2001"), "a", 1L}, new Object[]{T("2001"), "abc", 1L} + ) : + ImmutableList.of( + new Object[]{T("2000"), null, 1L}, + new Object[]{T("2000"), "", 1L}, + new Object[]{T("2000"), "a", 1L}, + new Object[]{T("2001"), null, 1L}, + new Object[]{T("2001"), "a", 1L}, + new Object[]{T("2001"), "abc", 1L} ) ); } @@ -5211,7 +5452,7 @@ public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Excepti public void testGroupByStringLength() throws Exception { testQuery( - "SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1)", + "SELECT CHARACTER_LENGTH(dim1), COUNT(*) FROM druid.foo GROUP BY CHARACTER_LENGTH(dim1) ORDER BY CHARACTER_LENGTH(dim1)", ImmutableList.of( GroupByQuery.builder() .setDataSource(CalciteTests.DATASOURCE1) @@ -5221,6 +5462,11 @@ public void testGroupByStringLength() throws Exception .setDimensions(DIMS(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG))) .setAggregatorSpecs(AGGS(new CountAggregatorFactory("a0"))) .setContext(QUERY_CONTEXT_DEFAULT) + .setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec( + "d0", + OrderByColumnSpec.Direction.ASCENDING, + StringComparators.NUMERIC + )), null)) .build() ), ImmutableList.of( @@ -5235,6 +5481,7 @@ public void testGroupByStringLength() throws Exception @Test public void testFilterAndGroupByLookup() throws Exception { + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn( null, "lookyloo", @@ -5279,7 +5526,7 @@ public void testFilterAndGroupByLookup() throws Exception .build() ), ImmutableList.of( - new Object[]{"", 5L}, + new Object[]{nullValue, 5L}, new Object[]{"xabc", 1L} ) ); @@ -5317,7 +5564,7 @@ public void testCountDistinctOfLookup() throws Exception .build() ), ImmutableList.of( - new Object[]{2L} + new Object[]{NullHandling.useDefaultValuesForNull() ? 2L : 1L} ) ); } @@ -5476,7 +5723,9 @@ public void testTimeseriesUsingTimeFloorWithTimeShift() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(timestamp_shift(\"__time\",'P1D',-1),'P1M','','UTC')", + "timestamp_floor(timestamp_shift(\"__time\",'P1D',-1),'P1M'," + + DruidExpression.nullLiteral() + + ",'UTC')", ValueType.LONG ) ) @@ -5524,7 +5773,7 @@ public void testTimeseriesUsingTimeFloorWithTimestampAdd() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor((\"__time\" + -86400000),'P1M','','UTC')", + "timestamp_floor((\"__time\" + -86400000),'P1M',null,'UTC')", ValueType.LONG ) ) @@ -5651,7 +5900,7 @@ public void testTimeseriesLosAngelesUsingTimeFloorConnectionLosAngeles() throws public void testTimeseriesDontSkipEmptyBuckets() throws Exception { // Tests that query context parameters are passed through to the underlying query engine. - + Long defaultVal = NullHandling.useDefaultValuesForNull() ? 0L : null; testQuery( PLANNER_CONFIG_DEFAULT, QUERY_CONTEXT_DONT_SKIP_EMPTY_BUCKETS, @@ -5673,29 +5922,29 @@ public void testTimeseriesDontSkipEmptyBuckets() throws Exception ), ImmutableList.builder() .add(new Object[]{1L, T("2000-01-01")}) - .add(new Object[]{0L, T("2000-01-01T01")}) - .add(new Object[]{0L, T("2000-01-01T02")}) - .add(new Object[]{0L, T("2000-01-01T03")}) - .add(new Object[]{0L, T("2000-01-01T04")}) - .add(new Object[]{0L, T("2000-01-01T05")}) - .add(new Object[]{0L, T("2000-01-01T06")}) - .add(new Object[]{0L, T("2000-01-01T07")}) - .add(new Object[]{0L, T("2000-01-01T08")}) - .add(new Object[]{0L, T("2000-01-01T09")}) - .add(new Object[]{0L, T("2000-01-01T10")}) - .add(new Object[]{0L, T("2000-01-01T11")}) - .add(new Object[]{0L, T("2000-01-01T12")}) - .add(new Object[]{0L, T("2000-01-01T13")}) - .add(new Object[]{0L, T("2000-01-01T14")}) - .add(new Object[]{0L, T("2000-01-01T15")}) - .add(new Object[]{0L, T("2000-01-01T16")}) - .add(new Object[]{0L, T("2000-01-01T17")}) - .add(new Object[]{0L, T("2000-01-01T18")}) - .add(new Object[]{0L, T("2000-01-01T19")}) - .add(new Object[]{0L, T("2000-01-01T20")}) - .add(new Object[]{0L, T("2000-01-01T21")}) - .add(new Object[]{0L, T("2000-01-01T22")}) - .add(new Object[]{0L, T("2000-01-01T23")}) + .add(new Object[]{defaultVal, T("2000-01-01T01")}) + .add(new Object[]{defaultVal, T("2000-01-01T02")}) + .add(new Object[]{defaultVal, T("2000-01-01T03")}) + .add(new Object[]{defaultVal, T("2000-01-01T04")}) + .add(new Object[]{defaultVal, T("2000-01-01T05")}) + .add(new Object[]{defaultVal, T("2000-01-01T06")}) + .add(new Object[]{defaultVal, T("2000-01-01T07")}) + .add(new Object[]{defaultVal, T("2000-01-01T08")}) + .add(new Object[]{defaultVal, T("2000-01-01T09")}) + .add(new Object[]{defaultVal, T("2000-01-01T10")}) + .add(new Object[]{defaultVal, T("2000-01-01T11")}) + .add(new Object[]{defaultVal, T("2000-01-01T12")}) + .add(new Object[]{defaultVal, T("2000-01-01T13")}) + .add(new Object[]{defaultVal, T("2000-01-01T14")}) + .add(new Object[]{defaultVal, T("2000-01-01T15")}) + .add(new Object[]{defaultVal, T("2000-01-01T16")}) + .add(new Object[]{defaultVal, T("2000-01-01T17")}) + .add(new Object[]{defaultVal, T("2000-01-01T18")}) + .add(new Object[]{defaultVal, T("2000-01-01T19")}) + .add(new Object[]{defaultVal, T("2000-01-01T20")}) + .add(new Object[]{defaultVal, T("2000-01-01T21")}) + .add(new Object[]{defaultVal, T("2000-01-01T22")}) + .add(new Object[]{defaultVal, T("2000-01-01T23")}) .build() ); } @@ -5891,7 +6140,7 @@ public void testGroupByExtractFloorTime() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_extract(timestamp_floor(\"__time\",'P1Y','','UTC'),'YEAR','UTC')", + "timestamp_extract(timestamp_floor(\"__time\",'P1Y',null,'UTC'),'YEAR','UTC')", ValueType.LONG ) ) @@ -5926,7 +6175,7 @@ public void testGroupByExtractFloorTimeLosAngeles() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_extract(timestamp_floor(\"__time\",'P1Y','','America/Los_Angeles'),'YEAR','America/Los_Angeles')", + "timestamp_extract(timestamp_floor(\"__time\",'P1Y',null,'America/Los_Angeles'),'YEAR','America/Los_Angeles')", ValueType.LONG ) ) @@ -5965,7 +6214,7 @@ public void testTimeseriesWithLimitNoTopN() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d0:v", - "timestamp_floor(\"__time\",'P1M','','UTC')", + "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG ) ) @@ -6009,7 +6258,7 @@ public void testTimeseriesWithLimit() throws Exception .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .virtualColumns( - EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M','','UTC')", ValueType.LONG) + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG) ) .dimension(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG)) .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) @@ -6042,7 +6291,7 @@ public void testTimeseriesWithOrderByAndLimit() throws Exception .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .virtualColumns( - EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M','','UTC')", ValueType.LONG) + EXPRESSION_VIRTUAL_COLUMN("d0:v", "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG) ) .dimension(new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG)) .aggregators(AGGS(new LongSumAggregatorFactory("a0", "cnt"))) @@ -6073,7 +6322,7 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setVirtualColumns( EXPRESSION_VIRTUAL_COLUMN( "d1:v", - "timestamp_floor(\"__time\",'P1M','','UTC')", + "timestamp_floor(\"__time\",'P1M',null,'UTC')", ValueType.LONG ) ) @@ -6100,12 +6349,21 @@ public void testGroupByTimeAndOtherDimension() throws Exception .setContext(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", T("2000-01-01"), 2L}, new Object[]{"", T("2001-01-01"), 1L}, new Object[]{"a", T("2000-01-01"), 1L}, new Object[]{"a", T("2001-01-01"), 1L}, new Object[]{"abc", T("2001-01-01"), 1L} + ) : + ImmutableList.of( + new Object[]{null, T("2000-01-01"), 1L}, + new Object[]{null, T("2001-01-01"), 1L}, + new Object[]{"", T("2000-01-01"), 1L}, + new Object[]{"a", T("2000-01-01"), 1L}, + new Object[]{"a", T("2001-01-01"), 1L}, + new Object[]{"abc", T("2001-01-01"), 1L} ) ); } @@ -6235,7 +6493,13 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) - .filters(AND(SELECTOR("dim1", "def", null), SELECTOR("dim2", "abc", null))) + .filters(OR( + SELECTOR("dim1", "def", null), + AND( + SELECTOR("dim1", "def", null), + SELECTOR("dim2", "abc", null) + ) + )) .columns("__time", "cnt", "dim1", "dim2") .resultFormat(ScanQuery.RESULT_FORMAT_COMPACTED_LIST) .context(QUERY_CONTEXT_DEFAULT) @@ -6250,8 +6514,9 @@ public void testUsingSubqueryAsFilterOnTwoColumns() throws Exception @Test public void testUsingSubqueryAsFilterWithInnerSort() throws Exception { - // Regression test for https://github.com/druid-io/druid/issues/4208 + String nullValue = NullHandling.useDefaultValuesForNull() ? "" : null; + // Regression test for https://github.com/druid-io/druid/issues/4208 testQuery( "SELECT dim1, dim2 FROM druid.foo\n" + " WHERE dim2 IN (\n" @@ -6288,13 +6553,20 @@ public void testUsingSubqueryAsFilterWithInnerSort() throws Exception .context(QUERY_CONTEXT_DEFAULT) .build() ), + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( new Object[]{"", "a"}, - new Object[]{"10.1", ""}, + new Object[]{"10.1", nullValue}, new Object[]{"2", ""}, new Object[]{"1", "a"}, new Object[]{"def", "abc"}, - new Object[]{"abc", ""} + new Object[]{"abc", nullValue} + ) : + ImmutableList.of( + new Object[]{"", "a"}, + new Object[]{"2", ""}, + new Object[]{"1", "a"}, + new Object[]{"def", "abc"} ) ); } @@ -6474,7 +6746,7 @@ private void verifyResults( Assert.assertEquals(StringUtils.format("result count: %s", sql), expectedResults.size(), results.size()); for (int i = 0; i < results.size(); i++) { Assert.assertArrayEquals( - StringUtils.format("result #%d: %s", i + 1, sql), + StringUtils.format("result #%d: %s obtainedResults: %s", i + 1, sql, results), expectedResults.get(i), results.get(i) ); diff --git a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java index 13d303124387..90f40f932125 100644 --- a/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/expression/ExpressionsTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.common.config.NullHandling; import io.druid.java.util.common.DateTimes; import io.druid.math.expr.ExprEval; import io.druid.math.expr.Parser; @@ -193,8 +194,8 @@ public void testStrpos() rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.VARCHAR)), rexBuilder.makeLiteral("ax") ), - DruidExpression.fromExpression("(strpos('','ax') + 1)"), - 0L + DruidExpression.fromExpression("(strpos(" + DruidExpression.nullLiteral() + ",'ax') + 1)"), + NullHandling.useDefaultValuesForNull() ? 0L : null ); } @@ -327,7 +328,9 @@ public void testDateTrunc() rexBuilder.makeLiteral("hour"), timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")) ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H'," + + DruidExpression.nullLiteral() + + ",'UTC')"), DateTimes.of("2000-02-03T04:00:00").getMillis() ); @@ -337,7 +340,9 @@ public void testDateTrunc() rexBuilder.makeLiteral("DAY"), timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")) ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'P1D','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'P1D'," + + DruidExpression.nullLiteral() + + ",'UTC')"), DateTimes.of("2000-02-03T00:00:00").getMillis() ); } @@ -388,7 +393,7 @@ public void testTimeFloor() timestampLiteral(DateTimes.of("2000-02-03T04:05:06Z")), rexBuilder.makeLiteral("PT1H") ), - DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(949550706000,'PT1H',null,'UTC')"), DateTimes.of("2000-02-03T04:00:00").getMillis() ); @@ -400,7 +405,7 @@ public void testTimeFloor() rexBuilder.makeNullLiteral(typeFactory.createSqlType(SqlTypeName.TIMESTAMP)), rexBuilder.makeLiteral("America/Los_Angeles") ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D','','America/Los_Angeles')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'America/Los_Angeles')"), DateTimes.of("2000-02-02T08:00:00").getMillis() ); } @@ -416,7 +421,7 @@ public void testOtherTimeFloor() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1Y','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1Y',null,'UTC')"), DateTimes.of("2000").getMillis() ); } @@ -432,7 +437,7 @@ public void testOtherTimeCeil() inputRef("t"), rexBuilder.makeFlag(TimeUnitRange.YEAR) ), - DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y','','UTC')"), + DruidExpression.fromExpression("timestamp_ceil(\"t\",'P1Y',null,'UTC')"), DateTimes.of("2001").getMillis() ); } @@ -667,7 +672,7 @@ public void testCastAsTimestamp() ), DruidExpression.of( null, - "timestamp_parse(\"tstr\",'','UTC')" + "timestamp_parse(\"tstr\",null,'UTC')" ), DateTimes.of("2000-02-03T04:05:06Z").getMillis() ); @@ -714,7 +719,7 @@ public void testCastAsDate() typeFactory.createSqlType(SqlTypeName.DATE), inputRef("t") ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'UTC')"), DateTimes.of("2000-02-03").getMillis() ); @@ -724,7 +729,7 @@ public void testCastAsDate() inputRef("dstr") ), DruidExpression.fromExpression( - "timestamp_floor(timestamp_parse(\"dstr\",'','UTC'),'P1D','','UTC')" + "timestamp_floor(timestamp_parse(\"dstr\",null,'UTC'),'P1D',null,'UTC')" ), DateTimes.of("2000-02-03").getMillis() ); @@ -742,7 +747,7 @@ public void testCastFromDate() ) ), DruidExpression.fromExpression( - "timestamp_format(timestamp_floor(\"t\",'P1D','','UTC'),'yyyy-MM-dd','UTC')" + "timestamp_format(timestamp_floor(\"t\",'P1D',null,'UTC'),'yyyy-MM-dd','UTC')" ), "2000-02-03" ); @@ -755,7 +760,7 @@ public void testCastFromDate() inputRef("t") ) ), - DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D','','UTC')"), + DruidExpression.fromExpression("timestamp_floor(\"t\",'P1D',null,'UTC')"), DateTimes.of("2000-02-03").getMillis() ); } diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index fd12039df693..38d237e62e24 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -23,12 +23,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; import io.druid.math.expr.ExprMacroTable; import io.druid.query.QueryInterruptedException; import io.druid.query.ResourceLimitExceededException; +import io.druid.common.config.NullHandling; import io.druid.server.security.AllowAllAuthenticator; import io.druid.server.security.NoopEscalator; import io.druid.server.security.AuthConfig; @@ -215,10 +217,18 @@ public void testFieldAliasingGroupBy() throws Exception ).rhs; Assert.assertEquals( + NullHandling.useDefaultValuesForNull() ? ImmutableList.of( ImmutableMap.of("x", "", "y", ""), ImmutableMap.of("x", "a", "y", "a"), ImmutableMap.of("x", "abc", "y", "abc") + ) : + ImmutableList.of( + // x and y both should be null instead of empty string + Maps.transformValues(ImmutableMap.of("x", "", "y", ""), (val) -> null), + ImmutableMap.of("x", "", "y", ""), + ImmutableMap.of("x", "a", "y", "a"), + ImmutableMap.of("x", "abc", "y", "abc") ), rows ); diff --git a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java index a3301506e08f..3d2c5103acfa 100644 --- a/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/planner/CalcitesTest.java @@ -28,7 +28,6 @@ public class CalcitesTest @Test public void testEscapeStringLiteral() { - Assert.assertEquals("''", Calcites.escapeStringLiteral(null)); Assert.assertEquals("''", Calcites.escapeStringLiteral("")); Assert.assertEquals("'foo'", Calcites.escapeStringLiteral("foo")); Assert.assertEquals("'foo bar'", Calcites.escapeStringLiteral("foo bar")); diff --git a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java index 5db831df0327..781141f33ed5 100644 --- a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java @@ -137,7 +137,8 @@ public class CalciteTests public static final String FORBIDDEN_DATASOURCE = "forbiddenDatasource"; public static final String TEST_SUPERUSER_NAME = "testSuperuser"; - public static final AuthorizerMapper TEST_AUTHORIZER_MAPPER = new AuthorizerMapper(null) { + public static final AuthorizerMapper TEST_AUTHORIZER_MAPPER = new AuthorizerMapper(null) + { @Override public Authorizer getAuthorizer(String name) { @@ -162,6 +163,7 @@ public Access authorize( } }; public static final AuthenticatorMapper TEST_AUTHENTICATOR_MAPPER; + static { final Map defaultMap = Maps.newHashMap(); defaultMap.put(