Make float default representation for DoubleSum/Min/Max aggregators

docs/content/configuration/index.md

@@ -395,3 +395,14 @@ the following properties.
 <div class="note info">
 JavaScript-based functionality is disabled by default. Please refer to the Druid <a href="../development/javascript.html">JavaScript programming guide</a> for guidelines about using Druid's JavaScript functionality, including instructions on how to enable it.
 </div>
+
+### Double Column storage
+
+Druid's storage layer uses a 32-bit float representation to store columns created by the 
+doubleSum, doubleMin, and doubleMax aggregators at indexing time. To instead use 64-bit floats
+for these columns, please set the system-wide property `druid.indexing.doubleStorage=double`.
+This will become the default behavior in a future version of Druid.
+
+|Property|Description|Default|
+|--------|-----------|-------|
+|`druid.indexing.doubleStorage`|Set to "double" to use 64-bit double representation for double columns.|float|

examples/conf-quickstart/druid/_common/common.runtime.properties

@@ -116,3 +116,8 @@ druid.selectors.coordinator.serviceName=druid/coordinator
 druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"]
 druid.emitter=logging
 druid.emitter.logging.logLevel=info
+
+# Storage type of double columns
+# ommiting this will lead to index double as float at the storage layer
+
+druid.indexing.doubleStorage=double

examples/conf/druid/_common/common.runtime.properties

@@ -115,3 +115,8 @@ druid.selectors.coordinator.serviceName=druid/coordinator
 druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"]
 druid.emitter=logging
 druid.emitter.logging.logLevel=info
+
+# Storage type of double columns
+# ommiting this will lead to index double as float at the storage layer
+
+druid.indexing.doubleStorage=double

pom.xml

@@ -1036,6 +1036,8 @@
                         <!-- set heap size to work around https://github.com/travis-ci/travis-ci/issues/3396 -->
                         <argLine>-Xmx3000m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8
                             -Duser.timezone=UTC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+                            <!--@TODO After fixing https://github.com/druid-io/druid/issues/4964 remove this parameter-->
+                            -Ddruid.indexing.doubleStorage=double
                         </argLine>
                         <!-- our tests are very verbose, let's keep the volume down -->
                         <redirectTestOutputToFile>true</redirectTestOutputToFile>
@@ -1264,7 +1266,9 @@
                             <!-- locale settings must be set on the command line before startup -->
                             <!-- set heap size to work around https://github.com/travis-ci/travis-ci/issues/3396 -->
                             <argLine>-Xmx768m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8
-                                -Duser.timezone=UTC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+                              -Duser.timezone=UTC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+                              <!--@TODO After fixing https://github.com/druid-io/druid/issues/4964 remove this parameter-->
+                              -Ddruid.indexing.doubleStorage=double
                             </argLine>
                             <!-- our tests are very verbose, let's keep the volume down -->
                             <redirectTestOutputToFile>true</redirectTestOutputToFile>

processing/src/main/java/io/druid/query/aggregation/SimpleDoubleAggregatorFactory.java

@@ -26,6 +26,7 @@
 import io.druid.math.expr.Parser;
 import io.druid.segment.BaseDoubleColumnValueSelector;
 import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.column.Column;
 
 import java.util.Collections;
 import java.util.Comparator;
@@ -38,6 +39,7 @@ public abstract class SimpleDoubleAggregatorFactory extends AggregatorFactory
   protected final String fieldName;
   protected final String expression;
   protected final ExprMacroTable macroTable;
+  protected final boolean storeDoubleAsFloat;
 
   public SimpleDoubleAggregatorFactory(
       ExprMacroTable macroTable,
@@ -50,6 +52,7 @@ public SimpleDoubleAggregatorFactory(
     this.fieldName = fieldName;
     this.name = name;
     this.expression = expression;
+    this.storeDoubleAsFloat = Column.storeDoubleAsFloat();
     Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name");
     Preconditions.checkArgument(
         fieldName == null ^ expression == null,
@@ -81,6 +84,9 @@ public Object deserialize(Object object)
   @Override
   public String getTypeName()
   {
+    if (storeDoubleAsFloat) {
+      return "float";
+    }
     return "double";
   }
 
@@ -144,4 +150,5 @@ public String getExpression()
   {
     return expression;
   }
+
 }

processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java

@@ -59,6 +59,7 @@ public class DoubleFirstAggregatorFactory extends AggregatorFactory
 
   private final String fieldName;
   private final String name;
+  private final boolean storeDoubleAsFloat;
 
   @JsonCreator
   public DoubleFirstAggregatorFactory(
@@ -71,6 +72,7 @@ public DoubleFirstAggregatorFactory(
 
     this.name = name;
     this.fieldName = fieldName;
+    this.storeDoubleAsFloat = Column.storeDoubleAsFloat();
   }
 
   @Override
@@ -222,6 +224,9 @@ public byte[] getCacheKey()
   @Override
   public String getTypeName()
   {
+    if (storeDoubleAsFloat) {
+      return "float";
+    }
     return "double";
   }
 

processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java

@@ -51,6 +51,7 @@ public class DoubleLastAggregatorFactory extends AggregatorFactory
 
   private final String fieldName;
   private final String name;
+  private final boolean storeDoubleAsFloat;
 
   @JsonCreator
   public DoubleLastAggregatorFactory(
@@ -62,6 +63,7 @@ public DoubleLastAggregatorFactory(
     Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName");
     this.name = name;
     this.fieldName = fieldName;
+    this.storeDoubleAsFloat = Column.storeDoubleAsFloat();
   }
 
   @Override
@@ -213,6 +215,10 @@ public byte[] getCacheKey()
   @Override
   public String getTypeName()
   {
+
+    if (storeDoubleAsFloat) {
+      return "float";
+    }
     return "double";
   }
 

processing/src/main/java/io/druid/segment/column/Column.java

@@ -19,11 +19,21 @@
 
 package io.druid.segment.column;
 
+import io.druid.java.util.common.StringUtils;
+
 /**
  */
 public interface Column
 {
   String TIME_COLUMN_NAME = "__time";
+  String DOUBLE_STORAGE_TYPE_PROPERTY = "druid.indexing.doubleStorage";
+
+  static boolean storeDoubleAsFloat()
+  {
+    String value = System.getProperty(DOUBLE_STORAGE_TYPE_PROPERTY, "float");
+    return !StringUtils.toLowerCase(value).equals("double");
+  }
+
   ColumnCapabilities getCapabilities();
 
   int getLength();