ARRAY_AGG sql aggregator function

core/src/main/java/org/apache/druid/math/expr/ExprEval.java

@@ -374,21 +374,11 @@ private static Class convertType(@Nullable Class existing, Class next)
     throw new UOE("Invalid array expression type: %s", next);
   }
 
-  public static ExprEval ofLong(@Nullable Number longValue)
-  {
-    return new LongExprEval(longValue);
-  }
-
   public static ExprEval of(long longValue)
   {
     return new LongExprEval(longValue);
   }
 
-  public static ExprEval ofDouble(@Nullable Number doubleValue)
-  {
-    return new DoubleExprEval(doubleValue);
-  }
-
   public static ExprEval of(double doubleValue)
   {
     return new DoubleExprEval(doubleValue);
@@ -402,22 +392,50 @@ public static ExprEval of(@Nullable String stringValue)
     return new StringExprEval(stringValue);
   }
 
+  public static ExprEval ofLong(@Nullable Number longValue)
+  {
+    if (longValue == null) {
+      return LongExprEval.OF_NULL;
+    }
+    return new LongExprEval(longValue);
+  }
+
+  public static ExprEval ofDouble(@Nullable Number doubleValue)
+  {
+    if (doubleValue == null) {
+      return DoubleExprEval.OF_NULL;
+    }
+    return new DoubleExprEval(doubleValue);
+  }
+
   public static ExprEval ofLongArray(@Nullable Long[] longValue)
   {
+    if (longValue == null) {
+      return LongArrayExprEval.OF_NULL;
+    }
     return new LongArrayExprEval(longValue);
   }
 
   public static ExprEval ofDoubleArray(@Nullable Double[] doubleValue)
   {
+    if (doubleValue == null) {
+      return DoubleArrayExprEval.OF_NULL;
+    }
     return new DoubleArrayExprEval(doubleValue);
   }
 
   public static ExprEval ofStringArray(@Nullable String[] stringValue)
   {
+    if (stringValue == null) {
+      return StringArrayExprEval.OF_NULL;
+    }
     return new StringArrayExprEval(stringValue);
   }
 
-  public static ExprEval of(boolean value, ExprType type)
+  /**
+   * Convert a boolean back into native expression type
+   */
+  public static ExprEval ofBoolean(boolean value, ExprType type)
   {
     switch (type) {
       case DOUBLE:
@@ -431,11 +449,17 @@ public static ExprEval of(boolean value, ExprType type)
     }
   }
 
+  /**
+   * Convert a boolean into a long expression type
+   */
   public static ExprEval ofLongBoolean(boolean value)
   {
     return ExprEval.of(Evals.asLong(value));
   }
 
+  /**
+   * Examine java type to find most appropriate expression type
+   */
   public static ExprEval bestEffortOf(@Nullable Object val)
   {
     if (val instanceof ExprEval) {
@@ -631,6 +655,8 @@ public boolean isNumericNull()
 
   private static class DoubleExprEval extends NumericExprEval
   {
+    private static final DoubleExprEval OF_NULL = new DoubleExprEval(null);
+
     private DoubleExprEval(@Nullable Number value)
     {
       super(value == null ? NullHandling.defaultDoubleValue() : (Double) value.doubleValue());
@@ -691,6 +717,8 @@ public Expr toExpr()
 
   private static class LongExprEval extends NumericExprEval
   {
+    private static final LongExprEval OF_NULL = new LongExprEval(null);
+
     private LongExprEval(@Nullable Number value)
     {
       super(value == null ? NullHandling.defaultLongValue() : (Long) value.longValue());
@@ -758,6 +786,8 @@ public Expr toExpr()
 
   private static class StringExprEval extends ExprEval<String>
   {
+    private static final StringExprEval OF_NULL = new StringExprEval(null);
+
     // Cached primitive values.
     private boolean intValueValid = false;
     private boolean longValueValid = false;
@@ -768,8 +798,6 @@ private static class StringExprEval extends ExprEval<String>
     private double doubleValue;
     private boolean booleanValue;
 
-    private static final StringExprEval OF_NULL = new StringExprEval(null);
-
     @Nullable
     private Number numericVal;
 
@@ -1014,6 +1042,8 @@ public T getIndex(int index)
 
   private static class LongArrayExprEval extends ArrayExprEval<Long>
   {
+    private static final LongArrayExprEval OF_NULL = new LongArrayExprEval(null);
+
     private LongArrayExprEval(@Nullable Long[] value)
     {
       super(value);
@@ -1073,6 +1103,8 @@ public Expr toExpr()
 
   private static class DoubleArrayExprEval extends ArrayExprEval<Double>
   {
+    private static final DoubleArrayExprEval OF_NULL = new DoubleArrayExprEval(null);
+
     private DoubleArrayExprEval(@Nullable Double[] value)
     {
       super(value);
@@ -1132,6 +1164,8 @@ public Expr toExpr()
 
   private static class StringArrayExprEval extends ArrayExprEval<String>
   {
+    private static final StringArrayExprEval OF_NULL = new StringArrayExprEval(null);
+
     private boolean longValueValid = false;
     private boolean doubleValueValid = false;
     private Long[] longValues;

core/src/main/java/org/apache/druid/math/expr/InputBindings.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.math.expr;
+
+import com.google.common.base.Supplier;
+
+import javax.annotation.Nullable;
+import java.util.Map;
+
+public class InputBindings
+{
+  /**
+   * Create an {@link Expr.InputBindingInspector} backed by a map of binding identifiers to their {@link ExprType}
+   */
+  public static Expr.InputBindingInspector inspectorFromTypeMap(final Map<String, ExprType> types)
+  {
+    return new Expr.InputBindingInspector()
+    {
+      @Nullable
+      @Override
+      public ExprType getType(String name)
+      {
+        return types.get(name);
+      }
+    };
+  }
+
+  /**
+   * Create {@link Expr.ObjectBinding} backed by {@link Map} to provide values for identifiers to evaluate {@link Expr}
+   */
+  public static Expr.ObjectBinding withMap(final Map<String, ?> bindings)
+  {
+    return bindings::get;
+  }
+
+  /**
+   * Create {@link Expr.ObjectBinding} backed by map of {@link Supplier} to provide values for identifiers to evaluate
+   * {@link Expr}
+   */
+  public static Expr.ObjectBinding withSuppliers(final Map<String, Supplier<Object>> bindings)
+  {
+    return (String name) -> {
+      Supplier<Object> supplier = bindings.get(name);
+      return supplier == null ? null : supplier.get();
+    };
+  }
+}

core/src/main/java/org/apache/druid/math/expr/Parser.java

@@ -601,23 +601,4 @@ public static void validateExpr(Expr expression, Expr.BindingAnalysis bindingAna
     }
   }
 
-  /**
-   * Create {@link Expr.ObjectBinding} backed by {@link Map} to provide values for identifiers to evaluate {@link Expr}
-   */
-  public static Expr.ObjectBinding withMap(final Map<String, ?> bindings)
-  {
-    return bindings::get;
-  }
-
-  /**
-   * Create {@link Expr.ObjectBinding} backed by map of {@link Supplier} to provide values for identifiers to evaluate
-   * {@link Expr}
-   */
-  public static Expr.ObjectBinding withSuppliers(final Map<String, Supplier<Object>> bindings)
-  {
-    return (String name) -> {
-      Supplier<Object> supplier = bindings.get(name);
-      return supplier == null ? null : supplier.get();
-    };
-  }
 }

core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java

@@ -166,7 +166,7 @@ public ExprEval eval(ObjectBinding bindings)
     }
     // conforming to other boolean-returning binary operators
     ExprType retType = ret.type() == ExprType.DOUBLE ? ExprType.DOUBLE : ExprType.LONG;
-    return ExprEval.of(!ret.asBoolean(), retType);
+    return ExprEval.ofBoolean(!ret.asBoolean(), retType);
   }
 
   @Nullable

core/src/test/java/org/apache/druid/math/expr/ApplyFunctionTest.java

@@ -48,7 +48,7 @@ public void setup()
     builder.put("d", new String[] {null});
     builder.put("e", new String[] {null, "foo", "bar"});
     builder.put("f", new String[0]);
-    bindings = Parser.withMap(builder.build());
+    bindings = InputBindings.withMap(builder.build());
   }
 
   @Test

core/src/test/java/org/apache/druid/math/expr/EvalTest.java

@@ -51,7 +51,7 @@ private ExprEval eval(String x, Expr.ObjectBinding bindings)
   @Test
   public void testDoubleEval()
   {
-    Expr.ObjectBinding bindings = Parser.withMap(ImmutableMap.of("x", 2.0d));
+    Expr.ObjectBinding bindings = InputBindings.withMap(ImmutableMap.of("x", 2.0d));
     Assert.assertEquals(2.0, evalDouble("x", bindings), 0.0001);
     Assert.assertEquals(2.0, evalDouble("\"x\"", bindings), 0.0001);
     Assert.assertEquals(304.0, evalDouble("300 + \"x\" * 2", bindings), 0.0001);
@@ -89,7 +89,7 @@ public void testDoubleEval()
   @Test
   public void testLongEval()
   {
-    Expr.ObjectBinding bindings = Parser.withMap(ImmutableMap.of("x", 9223372036854775807L));
+    Expr.ObjectBinding bindings = InputBindings.withMap(ImmutableMap.of("x", 9223372036854775807L));
 
     Assert.assertEquals(9223372036854775807L, evalLong("x", bindings));
     Assert.assertEquals(9223372036854775807L, evalLong("\"x\"", bindings));
@@ -147,7 +147,7 @@ public void testLongEval()
   @Test
   public void testBooleanReturn()
   {
-    Expr.ObjectBinding bindings = Parser.withMap(
+    Expr.ObjectBinding bindings = InputBindings.withMap(
         ImmutableMap.of("x", 100L, "y", 100L, "z", 100D, "w", 100D)
     );
     ExprEval eval = Parser.parse("x==y", ExprMacroTable.nil()).eval(bindings);

core/src/test/java/org/apache/druid/math/expr/FunctionTest.java

@@ -59,7 +59,7 @@ public void setup()
         .put("a", new String[] {"foo", "bar", "baz", "foobar"})
         .put("b", new Long[] {1L, 2L, 3L, 4L, 5L})
         .put("c", new Double[] {3.1, 4.2, 5.3});
-    bindings = Parser.withMap(builder.build());
+    bindings = InputBindings.withMap(builder.build());
   }
 
   @Test

core/src/test/java/org/apache/druid/math/expr/ParserTest.java

@@ -741,23 +741,23 @@ private void validateConstantExpression(String expression, Object expected)
     Assert.assertEquals(
         expression,
         expected,
-        parsed.eval(Parser.withMap(ImmutableMap.of())).value()
+        parsed.eval(InputBindings.withMap(ImmutableMap.of())).value()
     );
 
     final Expr parsedNoFlatten = Parser.parse(expression, ExprMacroTable.nil(), false);
     Expr parsedRoundTrip = Parser.parse(parsedNoFlatten.stringify(), ExprMacroTable.nil());
     Assert.assertEquals(
         expression,
         expected,
-        parsedRoundTrip.eval(Parser.withMap(ImmutableMap.of())).value()
+        parsedRoundTrip.eval(InputBindings.withMap(ImmutableMap.of())).value()
     );
     Assert.assertEquals(parsed.stringify(), parsedRoundTrip.stringify());
   }
 
   private void validateConstantExpression(String expression, Object[] expected)
   {
     Expr parsed = Parser.parse(expression, ExprMacroTable.nil());
-    Object evaluated = parsed.eval(Parser.withMap(ImmutableMap.of())).value();
+    Object evaluated = parsed.eval(InputBindings.withMap(ImmutableMap.of())).value();
     Assert.assertArrayEquals(
         expression,
         expected,
@@ -770,7 +770,7 @@ private void validateConstantExpression(String expression, Object[] expected)
     Assert.assertArrayEquals(
         expression,
         expected,
-        (Object[]) roundTrip.eval(Parser.withMap(ImmutableMap.of())).value()
+        (Object[]) roundTrip.eval(InputBindings.withMap(ImmutableMap.of())).value()
     );
     Assert.assertEquals(parsed.stringify(), roundTrip.stringify());
   }

docs/querying/sql.md

@@ -311,7 +311,7 @@ Aggregation functions can appear in the SELECT clause of any query. Any aggregat
 `AGG(expr) FILTER(WHERE whereExpr)`. Filtered aggregators will only aggregate rows that match their filter. It's
 possible for two aggregators in the same SQL query to have different filters.
 
-Only the COUNT aggregation can accept DISTINCT.
+Only the COUNT and ARRAY_AGG aggregations can accept DISTINCT.
 
 > The order of aggregation operations across segments is not deterministic. This means that non-commutative aggregation
 > functions can produce inconsistent results across the same query. 
@@ -353,6 +353,8 @@ Only the COUNT aggregation can accept DISTINCT.
 |`ANY_VALUE(expr)`|Returns any value of `expr` including null. `expr` must be numeric. This aggregator can simplify and optimize the performance by returning the first encountered value (including null)|
 |`ANY_VALUE(expr, maxBytesPerString)`|Like `ANY_VALUE(expr)`, but for strings. The `maxBytesPerString` parameter determines how much aggregation space to allocate per string. Strings longer than this limit will be truncated. This parameter should be set as low as possible, since high values will lead to wasted memory.|
 |`GROUPING(expr, expr...)`|Returns a number to indicate which groupBy dimension is included in a row, when using `GROUPING SETS`. Refer to [additional documentation](aggregations.md#grouping-aggregator) on how to infer this number.|
+|`ARRAY_AGG(expr, [size])`|Collects all values of `expr` into an ARRAY, including null values, with `size` in bytes limit on aggregation size (default of 1024 bytes). Use of `ORDER BY` within the `ARRAY_AGG` expression is not currently supported, and the ordering of results within the output array may vary depending on processing order.|
+|`ARRAY_AGG(DISTINCT expr, [size])`|Collects all distinct values of `expr` into an ARRAY, including null values, with `size` in bytes limit on aggregation size (default of 1024 bytes) per aggregate. Use of `ORDER BY` within the `ARRAY_AGG` expression is not currently supported, and the ordering of results within the output array may vary depending on processing order.|
 
 For advice on choosing approximate aggregation functions, check out our [approximate aggregations documentation](aggregations.md#approx).