SQL compatible Null Handling Part 1 - Expressions and Storage Changes

common/src/main/antlr4/io/druid/math/expr/antlr/Expr.g4

@@ -1,6 +1,7 @@
 grammar Expr;
 
-expr : ('-'|'!') expr                                 # unaryOpExpr
+expr : 'null'                                         # null
+     | ('-'|'!') expr                                 # unaryOpExpr
      |<assoc=right> expr '^' expr                     # powOpExpr
      | expr ('*'|'/'|'%') expr                        # mulDivModuloExpr
      | expr ('+'|'-') expr                            # addSubExpr

common/src/main/java/io/druid/common/config/NullHandling.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.common.config;
+
+import com.google.common.base.Strings;
+import com.google.inject.Inject;
+
+import javax.annotation.Nullable;
+
+/**
+ * Helper class for NullHandling. This class is used to switch between SQL compatible Null Handling behavior
+ * introduced as part of https://github.com/druid-io/druid/issues/4349 and the old druid behavior
+ * where null values are replaced with default values e.g Null Strings are replaced with empty values.
+ */
+public class NullHandling
+{
+  public static String NULL_HANDLING_CONFIG_STRING = "druid.generic.useDefaultValueForNull";
+
+  /**
+   * use these values to ensure that {@link NullHandling#defaultDoubleValue()},
+   * {@link NullHandling#defaultFloatValue()} , {@link NullHandling#defaultFloatValue()}
+   * return the same boxed object when returning a constant zero
+   */
+  public static final Double ZERO_DOUBLE = 0.0d;
+  public static final Float ZERO_FLOAT = 0.0f;
+  public static final Long ZERO_LONG = 0L;
+
+  /**
+   * INSTANCE is injected using static injection to avoid adding JacksonInject annotations all over the code.
+   * See io.druid.guice.NullHandlingModule for details.
+   * It does not take effect in all unit tests since we don't use Guice Injection.
+   * For tests default system property is supposed to be used only in tests
+   */
+  @Inject
+  private static NullValueHandlingConfig INSTANCE = new NullValueHandlingConfig(
+      Boolean.valueOf(System.getProperty(NULL_HANDLING_CONFIG_STRING, "true"))
+  );
+
+  public static boolean replaceWithDefault()
+  {
+    return INSTANCE.isUseDefaultValuesForNull();
+  }
+
+  public static boolean sqlCompatible()
+  {
+    return !replaceWithDefault();
+  }
+
+  @Nullable
+  public static String nullToEmptyIfNeeded(@Nullable String value)
+  {
+    //CHECKSTYLE.OFF: Regexp
+    return replaceWithDefault() ? Strings.nullToEmpty(value) : value;
+    //CHECKSTYLE.ON: Regexp
+  }
+
+  @Nullable
+  public static String emptyToNullIfNeeded(@Nullable String value)
+  {
+    //CHECKSTYLE.OFF: Regexp
+    return replaceWithDefault() ? Strings.emptyToNull(value) : value;
+    //CHECKSTYLE.ON: Regexp
+  }
+
+  @Nullable
+  public static String defaultStringValue()
+  {
+    return replaceWithDefault() ? "" : null;
+  }
+
+  @Nullable
+  public static Long defaultLongValue()
+  {
+    return replaceWithDefault() ? ZERO_LONG : null;
+  }
+
+  @Nullable
+  public static Float defaultFloatValue()
+  {
+    return replaceWithDefault() ? ZERO_FLOAT : null;
+  }
+
+  @Nullable
+  public static Double defaultDoubleValue()
+  {
+    return replaceWithDefault() ? ZERO_DOUBLE : null;
+  }
+
+  public static boolean isNullOrEquivalent(@Nullable String value)
+  {
+    return replaceWithDefault() ? Strings.isNullOrEmpty(value) : value == null;
+  }
+
+}

common/src/main/java/io/druid/common/config/NullValueHandlingConfig.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.common.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class NullValueHandlingConfig
+{
+
+  @JsonProperty("useDefaultValueForNull")
+  private final boolean useDefaultValuesForNull;
+
+  @JsonCreator
+  public NullValueHandlingConfig(@JsonProperty("useDefaultValueForNull") Boolean useDefaultValuesForNull)
+  {
+    this.useDefaultValuesForNull = useDefaultValuesForNull == null
+                                   ? true
+                                   : useDefaultValuesForNull;
+  }
+
+  public boolean isUseDefaultValuesForNull()
+  {
+    return useDefaultValuesForNull;
+  }
+}

common/src/main/java/io/druid/common/utils/SerializerUtils.java

@@ -142,7 +142,7 @@ private void writeInt(OutputStream out, int intValue) throws IOException
     out.write(Ints.toByteArray(intValue));
   }
 
-  private void writeInt(WritableByteChannel out, int intValue) throws IOException
+  public static void writeInt(WritableByteChannel out, int intValue) throws IOException
   {
     final ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
     buffer.putInt(intValue);

common/src/main/java/io/druid/math/expr/Expr.java

@@ -20,9 +20,9 @@
 package io.druid.math.expr;
 
 import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
 import com.google.common.math.LongMath;
 import com.google.common.primitives.Ints;
+import io.druid.common.config.NullHandling;
 import io.druid.java.util.common.IAE;
 import io.druid.java.util.common.ISE;
 import io.druid.java.util.common.guava.Comparators;
@@ -124,7 +124,7 @@ class StringExpr extends ConstantExpr
 
   public StringExpr(String value)
   {
-    this.value = Strings.emptyToNull(value);
+    this.value = NullHandling.emptyToNullIfNeeded(value);
   }
 
   @Nullable
@@ -362,6 +362,13 @@ public ExprEval eval(ObjectBinding bindings)
   {
     ExprEval leftVal = left.eval(bindings);
     ExprEval rightVal = right.eval(bindings);
+
+    // Result of any Binary expressions is null if any of the argument is null.
+    // e.g "select null * 2 as c;" or "select null + 1 as c;" will return null as per Standard SQL spec.
+    if (NullHandling.sqlCompatible() && (leftVal.isNull() || rightVal.isNull())) {
+      return ExprEval.of(null);
+    }
+
     if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) {
       return evalString(leftVal.asString(), rightVal.asString());
     } else if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) {
@@ -491,7 +498,8 @@ class BinPlusExpr extends BinaryEvalOpExprBase
   @Override
   protected ExprEval evalString(@Nullable String left, @Nullable String right)
   {
-    return ExprEval.of(Strings.nullToEmpty(left) + Strings.nullToEmpty(right));
+    return ExprEval.of(NullHandling.nullToEmptyIfNeeded(left)
+                       + NullHandling.nullToEmptyIfNeeded(right));
   }
 
   @Override

common/src/main/java/io/druid/math/expr/ExprEval.java

@@ -20,12 +20,14 @@
 package io.druid.math.expr;
 
 import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
 import com.google.common.primitives.Doubles;
 import com.google.common.primitives.Ints;
+import io.druid.common.config.NullHandling;
 import io.druid.common.guava.GuavaUtils;
 import io.druid.java.util.common.IAE;
 
+import javax.annotation.Nullable;
+
 /**
  */
 public abstract class ExprEval<T>
@@ -50,7 +52,7 @@ public static ExprEval of(double doubleValue)
     return new DoubleExprEval(doubleValue);
   }
 
-  public static ExprEval of(String stringValue)
+  public static ExprEval of(@Nullable String stringValue)
   {
     return new StringExprEval(stringValue);
   }
@@ -108,6 +110,7 @@ public boolean isNull()
 
   public abstract double asDouble();
 
+  @Nullable
   public String asString()
   {
     return value == null ? null : String.valueOf(value);
@@ -228,9 +231,9 @@ public Expr toExpr()
 
   private static class StringExprEval extends ExprEval<String>
   {
-    private StringExprEval(String value)
+    private StringExprEval(@Nullable String value)
     {
-      super(Strings.emptyToNull(value));
+      super(NullHandling.emptyToNullIfNeeded(value));
     }
 
     @Override
@@ -243,10 +246,12 @@ public final ExprType type()
     public final int asInt()
     {
       if (value == null) {
+        assert NullHandling.replaceWithDefault();
         return 0;
       }
 
       final Integer theInt = Ints.tryParse(value);
+      assert NullHandling.replaceWithDefault() || theInt != null;
       return theInt == null ? 0 : theInt;
     }
 
@@ -255,17 +260,20 @@ public final long asLong()
     {
       // GuavaUtils.tryParseLong handles nulls, no need for special null handling here.
       final Long theLong = GuavaUtils.tryParseLong(value);
+      assert NullHandling.replaceWithDefault() || theLong != null;
       return theLong == null ? 0L : theLong;
     }
 
     @Override
     public final double asDouble()
     {
       if (value == null) {
+        assert NullHandling.replaceWithDefault();
         return 0.0;
       }
 
       final Double theDouble = Doubles.tryParse(value);
+      assert NullHandling.replaceWithDefault() || theDouble != null;
       return theDouble == null ? 0.0 : theDouble;
     }
 

common/src/main/java/io/druid/math/expr/ExprListenerImpl.java

@@ -335,4 +335,10 @@ public void exitFunctionArgs(ExprParser.FunctionArgsContext ctx)
 
     nodes.put(ctx, args);
   }
+
+  @Override
+  public void exitNull(ExprParser.NullContext ctx)
+  {
+    nodes.put(ctx, new StringExpr(null));
+  }
 }