From fea7cd66b5d97c17b8ae539be22702893f066318 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 9 Feb 2023 03:18:16 -0800 Subject: [PATCH 1/7] fix array_agg to work with complex types and bugs with expression aggregator complex array handling --- .../org/apache/druid/math/expr/ExprEval.java | 13 ++- .../ObjectStrategyComplexTypeStrategy.java | 6 +- .../builtin/ArraySqlAggregator.java | 5 -- .../calcite/CalciteNestedDataQueryTest.java | 84 +++++++++++++++++++ 4 files changed, 98 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 1d5298ffc1d4..3f6211e56030 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -329,7 +329,7 @@ public static ExprEval ofStringArray(@Nullable Object[] stringValue) } - public static ExprEval ofArray(ExpressionType outputType, Object[] value) + public static ExprEval ofArray(ExpressionType outputType, @Nullable Object[] value) { Preconditions.checkArgument(outputType.isArray(), "Output type %s is not an array", outputType); return new ArrayExprEval(outputType, value); @@ -572,6 +572,9 @@ public static ExprEval ofType(@Nullable ExpressionType type, @Nullable Object va if (type.getElementType().isArray()) { return ofArray(type, (Object[]) value); } + if (value == null) { + return ofArray(type, null); + } // in a better world, we might get an object that matches the type signature for arrays and could do a switch // statement here, but this is not that world yet, and things that are array typed might also be non-arrays, // e.g. we might get a String instead of String[], so just fallback to bestEffortOf @@ -798,7 +801,7 @@ public final ExprEval castTo(ExpressionType castTo) return ExprEval.ofStringArray(value == null ? null : new Object[] {value.toString()}); } } - throw new IAE("invalid type " + castTo); + throw new IAE("invalid type cannot cast " + ExpressionType.DOUBLE + " to " + castTo); } @Override @@ -1308,7 +1311,11 @@ public ExprEval castTo(ExpressionType castTo) if (expressionType.equals(castTo)) { return this; } - throw new IAE("invalid type " + castTo); + // allow cast of unknown complex to some other complex type + if (expressionType.getComplexTypeName() == null) { + return new ComplexExprEval(castTo, value); + } + throw new IAE("invalid type cannot cast " + expressionType + " to " + castTo); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java index 93b992a4f428..d40ee5dee8e1 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ObjectStrategyComplexTypeStrategy.java @@ -46,7 +46,7 @@ public ObjectStrategyComplexTypeStrategy(ObjectStrategy objectStrategy, TypeS public int estimateSizeBytes(@Nullable T value) { byte[] bytes = objectStrategy.toBytes(value); - return bytes == null ? 0 : bytes.length; + return Integer.BYTES + (bytes == null ? 0 : bytes.length); } @Override @@ -56,7 +56,9 @@ public T read(ByteBuffer buffer) ByteBuffer dupe = buffer.duplicate(); dupe.order(buffer.order()); dupe.limit(dupe.position() + complexLength); - return objectStrategy.fromByteBuffer(dupe, complexLength); + T value = objectStrategy.fromByteBuffer(dupe, complexLength); + buffer.position(buffer.position() + complexLength); + return value; } @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java index edbf2b8da5d6..c3bf402df9e6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java @@ -48,9 +48,7 @@ import org.apache.druid.sql.calcite.expression.Expressions; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.PlannerContext; -import org.apache.druid.sql.calcite.planner.UnsupportedSQLQueryException; import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; -import org.apache.druid.sql.calcite.table.RowSignatures; import javax.annotation.Nullable; import java.util.List; @@ -167,9 +165,6 @@ static class ArrayAggReturnTypeInference implements SqlReturnTypeInference public RelDataType inferReturnType(SqlOperatorBinding sqlOperatorBinding) { RelDataType type = sqlOperatorBinding.getOperandType(0); - if (type instanceof RowSignatures.ComplexSqlType) { - throw new UnsupportedSQLQueryException("Cannot use ARRAY_AGG on complex inputs %s", type); - } return sqlOperatorBinding.getTypeFactory().createArrayType( type, -1 diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index f11a7c2717d0..840c1f963065 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -33,12 +33,14 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.Druids; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.ExpressionLambdaAggregatorFactory; import org.apache.druid.query.aggregation.FilteredAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; @@ -857,6 +859,88 @@ public void testGroupByPathSelectorFilter() ); } + @Test + public void testJsonAndArrayAgg() + { + cannotVectorize(); + testQuery( + "SELECT " + + "string, " + + "ARRAY_AGG(nest, 16384), " + + "SUM(cnt) " + + "FROM druid.nested GROUP BY 1", + ImmutableList.of( + GroupByQuery.builder() + .setDataSource(DATA_SOURCE) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setDimensions( + dimensions( + new DefaultDimensionSpec("string", "d0") + ) + ) + .setAggregatorSpecs( + aggregators( + /* + combineExpressionString='array_concat("__acc", "a0")', compareExpressionString='null', finalizeExpressionString='null', maxSizeBytes=16384} + */ + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("nest"), + "__acc", + "ARRAY>[]", + "ARRAY>[]", + true, + true, + false, + "array_append(\"__acc\", \"nest\")", + "array_concat(\"__acc\", \"a0\")", + null, + null, + HumanReadableBytes.valueOf(16384), + queryFramework().macroTable() + ), + new LongSumAggregatorFactory("a1", "cnt") + ) + ) + .setContext(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{ + "aaa", + "[{\"x\":100,\"y\":2.02,\"z\":\"300\",\"mixed\":1,\"mixed2\":\"1\"},{\"x\":100,\"y\":2.02,\"z\":\"400\",\"mixed2\":1.1}]", + 2L + }, + new Object[]{ + "bbb", + "[null]", + 1L + }, + new Object[]{ + "ccc", + "[{\"x\":200,\"y\":3.03,\"z\":\"abcdef\",\"mixed\":1.1,\"mixed2\":1}]", + 1L + }, + new Object[]{ + "ddd", + "[null,null]", + 2L + }, + new Object[]{ + "eee", + "[null]", + 1L + } + ), + RowSignature.builder() + .add("string", ColumnType.STRING) + .add("EXPR$1", ColumnType.ofArray(NestedDataComplexTypeSerde.TYPE)) + .add("EXPR$2", ColumnType.LONG) + .build() + ); + } + @Test public void testGroupByPathSelectorFilterLong() { From cdd19c6a59f6827440ff99afe60668ec2d189df6 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 9 Feb 2023 03:31:08 -0800 Subject: [PATCH 2/7] missed a few --- .../main/java/org/apache/druid/math/expr/ExprEval.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 3f6211e56030..826ae4710c9c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -801,7 +801,7 @@ public final ExprEval castTo(ExpressionType castTo) return ExprEval.ofStringArray(value == null ? null : new Object[] {value.toString()}); } } - throw new IAE("invalid type cannot cast " + ExpressionType.DOUBLE + " to " + castTo); + throw new IAE("invalid type cannot cast " + type() + " to " + castTo); } @Override @@ -866,7 +866,7 @@ public final ExprEval castTo(ExpressionType castTo) return ExprEval.ofStringArray(value == null ? null : new Object[] {value.toString()}); } } - throw new IAE("invalid type " + castTo); + throw new IAE("invalid type cannot cast " + type() + " to " + castTo); } @Override @@ -1035,7 +1035,7 @@ public final ExprEval castTo(ExpressionType castTo) return ExprEval.ofStringArray(value == null ? null : new Object[] {value}); } } - throw new IAE("invalid type " + castTo); + throw new IAE("invalid type cannot cast " + type() + " to " + castTo); } @Override @@ -1224,7 +1224,7 @@ public ExprEval castTo(ExpressionType castTo) return ExprEval.ofArray(castTo, cast); } - throw new IAE("invalid type " + castTo); + throw new IAE("invalid type cannot cast " + type() + " to " + castTo); } @Override From 96c56c1a24b6d2d767d44b1b3c7d00926f72a85f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 9 Feb 2023 04:33:24 -0800 Subject: [PATCH 3/7] update test --- .../org/apache/druid/math/expr/EvalTest.java | 77 +++++++++++-------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java index f1d3bee5b959..d3009ce4b585 100644 --- a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java @@ -29,9 +29,7 @@ import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.nio.ByteBuffer; import java.util.HashMap; @@ -54,9 +52,6 @@ public static void setupClass() ); } - @Rule - public ExpectedException expectedException = ExpectedException.none(); - private long evalLong(String x, Expr.ObjectBinding bindings) { ExprEval ret = eval(x, bindings); @@ -270,73 +265,91 @@ public void testArrayToScalar() @Test public void testStringArrayToScalarStringBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type STRING"); - ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.STRING); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.STRING) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to STRING", t.getMessage()); } @Test public void testStringArrayToScalarLongBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type LONG"); - ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.LONG); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.LONG) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to LONG", t.getMessage()); } @Test public void testStringArrayToScalarDoubleBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type DOUBLE"); - ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.DOUBLE); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofStringArray(new String[]{"foo", "bar"}).castTo(ExpressionType.DOUBLE) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to DOUBLE", t.getMessage()); } @Test public void testLongArrayToScalarStringBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type STRING"); - ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.STRING); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.STRING) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to STRING", t.getMessage()); } @Test public void testLongArrayToScalarLongBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type LONG"); - ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.LONG); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.LONG) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to LONG", t.getMessage()); } @Test public void testLongArrayToScalarDoubleBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type DOUBLE"); - ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.DOUBLE); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofLongArray(new Long[]{1L, 2L}).castTo(ExpressionType.DOUBLE) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to DOUBLE", t.getMessage()); } @Test public void testDoubleArrayToScalarStringBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type STRING"); - ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.STRING); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.STRING) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to STRING", t.getMessage()); } @Test public void testDoubleArrayToScalarLongBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type LONG"); - ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.LONG); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.LONG) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to LONG", t.getMessage()); } @Test public void testDoubleArrayToScalarDoubleBadCast() { - expectedException.expect(IAE.class); - expectedException.expectMessage("invalid type DOUBLE"); - ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.DOUBLE); + Throwable t = Assert.assertThrows( + IAE.class, + () -> ExprEval.ofDoubleArray(new Double[]{1.1, 2.2}).castTo(ExpressionType.DOUBLE) + ); + Assert.assertEquals("invalid type cannot cast ARRAY to DOUBLE", t.getMessage()); } @Test From b985c9a1ea75ca8366a805ece5674d2b9efc68c7 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 9 Feb 2023 05:22:53 -0800 Subject: [PATCH 4/7] fix test --- core/src/main/java/org/apache/druid/math/expr/ExprEval.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 826ae4710c9c..94d5d95ddd3f 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -1219,7 +1219,11 @@ public ExprEval castTo(ExpressionType castTo) ExpressionType elementType = (ExpressionType) castTo.getElementType(); Object[] cast = new Object[value.length]; for (int i = 0; i < value.length; i++) { - cast[i] = ExprEval.ofType(elementType(), value[i]).castTo(elementType).value(); + if (value[i] == null) { + cast[i] = null; + } else { + cast[i] = ExprEval.ofType(elementType(), value[i]).castTo(elementType).value(); + } } return ExprEval.ofArray(castTo, cast); } From 96d4084c871c49f6f88eca31c055930441700eaf Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 9 Feb 2023 14:33:36 -0800 Subject: [PATCH 5/7] more consistent handling of array expressions, numeric arrays more consistently honor druid.generic.useDefaultValueForNull, fix array_ordinal sql output type --- .../org/apache/druid/math/expr/ExprEval.java | 42 +++++++++------ .../org/apache/druid/math/expr/EvalTest.java | 22 +++++++- .../ArrayOrdinalOperatorConversion.java | 2 +- .../sql/calcite/CalciteArraysQueryTest.java | 51 ++++++++++++++----- .../druid/sql/calcite/CalciteQueryTest.java | 44 ++++++++++++---- 5 files changed, 120 insertions(+), 41 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 94d5d95ddd3f..dbfc11d3c4ee 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -170,7 +170,7 @@ public static NonnullPair coerceListToArray(@Nullable Object[] array = new Object[val.size()]; int i = 0; for (Object o : val) { - array[i++] = o == null ? null : ExprEval.ofType(ExpressionType.DOUBLE, o).value(); + array[i++] = ExprEval.ofType(ExpressionType.DOUBLE, o).value(); } return new NonnullPair<>(ExpressionType.DOUBLE_ARRAY, array); } else if (coercedType == Object.class) { @@ -193,7 +193,7 @@ public static NonnullPair coerceListToArray(@Nullable if (eval != null) { array[i++] = eval.castTo(elementType).value(); } else { - array[i++] = null; + array[i++] = ExprEval.ofType(elementType, null).value(); } } ExpressionType arrayType = elementType == null @@ -398,7 +398,7 @@ public static ExprEval bestEffortOf(@Nullable Object val) final Long[] inputArray = (Long[]) val; final Object[] array = new Object[inputArray.length]; for (int i = 0; i < inputArray.length; i++) { - array[i] = inputArray[i]; + array[i] = inputArray[i] != null ? inputArray[i] : NullHandling.defaultLongValue(); } return new ArrayExprEval(ExpressionType.LONG_ARRAY, array); } @@ -414,7 +414,7 @@ public static ExprEval bestEffortOf(@Nullable Object val) final Integer[] inputArray = (Integer[]) val; final Object[] array = new Object[inputArray.length]; for (int i = 0; i < inputArray.length; i++) { - array[i] = inputArray[i] == null ? null : inputArray[i].longValue(); + array[i] = inputArray[i] != null ? inputArray[i].longValue() : NullHandling.defaultLongValue(); } return new ArrayExprEval(ExpressionType.LONG_ARRAY, array); } @@ -430,7 +430,7 @@ public static ExprEval bestEffortOf(@Nullable Object val) final Double[] inputArray = (Double[]) val; final Object[] array = new Object[inputArray.length]; for (int i = 0; i < inputArray.length; i++) { - array[i] = inputArray[i]; + array[i] = inputArray[i] != null ? inputArray[i] : NullHandling.defaultDoubleValue(); } return new ArrayExprEval(ExpressionType.DOUBLE_ARRAY, array); } @@ -446,7 +446,7 @@ public static ExprEval bestEffortOf(@Nullable Object val) final Float[] inputArray = (Float[]) val; final Object[] array = new Object[inputArray.length]; for (int i = 0; i < inputArray.length; i++) { - array[i] = inputArray[i] != null ? inputArray[i].doubleValue() : null; + array[i] = inputArray[i] != null ? inputArray[i].doubleValue() : NullHandling.defaultDoubleValue(); } return new ArrayExprEval(ExpressionType.DOUBLE_ARRAY, array); } @@ -568,13 +568,29 @@ public static ExprEval ofType(@Nullable ExpressionType type, @Nullable Object va return ofComplex(type, value); case ARRAY: - // nested arrays, here be dragons... don't do any fancy coercion, assume everything is already sane types... - if (type.getElementType().isArray()) { - return ofArray(type, (Object[]) value); - } + ExpressionType elementType = (ExpressionType) type.getElementType(); if (value == null) { return ofArray(type, null); } + if (value instanceof List) { + List theList = (List) value; + Object[] array = new Object[theList.size()]; + int i = 0; + for (Object o : theList) { + array[i++] = ExprEval.ofType(elementType, o).value(); + } + return ofArray(type, array); + } + + if (value instanceof Object[]) { + Object[] inputArray = (Object[]) value; + Object[] array = new Object[inputArray.length]; + int i = 0; + for (Object o : inputArray) { + array[i++] = ExprEval.ofType(elementType, o).value(); + } + return ofArray(type, array); + } // in a better world, we might get an object that matches the type signature for arrays and could do a switch // statement here, but this is not that world yet, and things that are array typed might also be non-arrays, // e.g. we might get a String instead of String[], so just fallback to bestEffortOf @@ -1219,11 +1235,7 @@ public ExprEval castTo(ExpressionType castTo) ExpressionType elementType = (ExpressionType) castTo.getElementType(); Object[] cast = new Object[value.length]; for (int i = 0; i < value.length; i++) { - if (value[i] == null) { - cast[i] = null; - } else { - cast[i] = ExprEval.ofType(elementType(), value[i]).castTo(elementType).value(); - } + cast[i] = ExprEval.ofType(elementType(), value[i]).castTo(elementType).value(); } return ExprEval.ofArray(castTo, cast); } diff --git a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java index d3009ce4b585..d3b5760a8042 100644 --- a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java @@ -783,14 +783,26 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {"1", "2", "wat", "3"}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1L, 2L, 3L}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1L, 2L, null, 3L}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1.0, 2.0, 3.0}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1.0, 2.0, null, 3.0}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1.0, 2L, "3", true, false}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L, 1L, 0L}, (Object[]) eval.value()); @@ -800,10 +812,18 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {"1", "2", "wat", "3"}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, NullHandling.defaultDoubleValue(), 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1L, 2L, 3L}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1L, 2L, null, 3L}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, NullHandling.defaultDoubleValue(), 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1.0, 2.0, 3.0}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); @@ -922,7 +942,7 @@ public void testBestEffortOf() eval = ExprEval.bestEffortOf(new Object[] {null, 1.0, 2.0, 3.0}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); - Assert.assertArrayEquals(new Object[] {null, 1.0, 2.0, 3.0}, (Object[]) eval.value()); + Assert.assertArrayEquals(new Object[] {NullHandling.defaultDoubleValue(), 1.0, 2.0, 3.0}, (Object[]) eval.value()); eval = ExprEval.bestEffortOf(new double[] {1.0, 2.0, 3.0}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java index f2592d574d39..b758c2a89722 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOrdinalOperatorConversion.java @@ -62,7 +62,7 @@ public RelDataType inferReturnType(SqlOperatorBinding sqlOperatorBinding) { RelDataType type = sqlOperatorBinding.getOperandType(0); if (SqlTypeUtil.isArray(type)) { - type.getComponentType(); + return type.getComponentType(); } return type; } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index ead3b506e3cf..119cae8634ec 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -236,10 +236,10 @@ public void testSomeArrayFunctionsWithScanQuery() "[\"a\",\"a\",\"b\"]", "[7,0]", "[1.0,0.0]", - "7", - "1.0", - "7", - "1.0" + 7L, + 1.0, + 7L, + 1.0 } ); } else { @@ -264,10 +264,10 @@ public void testSomeArrayFunctionsWithScanQuery() "[\"a\",\"a\",\"b\"]", "[7,null]", "[1.0,null]", - "7", - "1.0", - "7", - "1.0" + 7L, + 1.0, + 7L, + 1.0 } ); } @@ -312,10 +312,10 @@ public void testSomeArrayFunctionsWithScanQuery() "array_concat(array(\"d1\"),array(\"d2\"))", ColumnType.DOUBLE_ARRAY ), - expressionVirtualColumn("v12", "array_offset(array(\"l1\"),0)", ColumnType.LONG_ARRAY), - expressionVirtualColumn("v13", "array_offset(array(\"d1\"),0)", ColumnType.DOUBLE_ARRAY), - expressionVirtualColumn("v14", "array_ordinal(array(\"l1\"),1)", ColumnType.LONG_ARRAY), - expressionVirtualColumn("v15", "array_ordinal(array(\"d1\"),1)", ColumnType.DOUBLE_ARRAY), + expressionVirtualColumn("v12", "array_offset(array(\"l1\"),0)", ColumnType.LONG), + expressionVirtualColumn("v13", "array_offset(array(\"d1\"),0)", ColumnType.DOUBLE), + expressionVirtualColumn("v14", "array_ordinal(array(\"l1\"),1)", ColumnType.LONG), + expressionVirtualColumn("v15", "array_ordinal(array(\"d1\"),1)", ColumnType.DOUBLE), expressionVirtualColumn("v2", "array(1.9,2.2,4.3)", ColumnType.DOUBLE_ARRAY), expressionVirtualColumn("v3", "array_append(\"dim3\",'foo')", ColumnType.STRING_ARRAY), expressionVirtualColumn("v4", "array_prepend('foo',array(\"dim2\"))", ColumnType.STRING_ARRAY), @@ -355,7 +355,32 @@ public void testSomeArrayFunctionsWithScanQuery() .context(QUERY_CONTEXT_DEFAULT) .build() ), - expectedResults + expectedResults, + RowSignature.builder() + .add("dim1", ColumnType.STRING) + .add("dim2", ColumnType.STRING) + .add("dim3", ColumnType.STRING) + .add("l1", ColumnType.LONG) + .add("l2", ColumnType.LONG) + .add("d1", ColumnType.DOUBLE) + .add("d2", ColumnType.DOUBLE) + .add("EXPR$7", ColumnType.STRING_ARRAY) + .add("EXPR$8", ColumnType.LONG_ARRAY) + .add("EXPR$9", ColumnType.DOUBLE_ARRAY) + .add("EXPR$10", ColumnType.STRING_ARRAY) + .add("EXPR$11", ColumnType.STRING_ARRAY) + .add("EXPR$12", ColumnType.LONG_ARRAY) + .add("EXPR$13", ColumnType.LONG_ARRAY) + .add("EXPR$14", ColumnType.DOUBLE_ARRAY) + .add("EXPR$15", ColumnType.DOUBLE_ARRAY) + .add("EXPR$16", ColumnType.STRING_ARRAY) + .add("EXPR$17", ColumnType.LONG_ARRAY) + .add("EXPR$18", ColumnType.DOUBLE_ARRAY) + .add("EXPR$19", ColumnType.LONG) + .add("EXPR$20", ColumnType.DOUBLE) + .add("EXPR$21", ColumnType.LONG) + .add("EXPR$22", ColumnType.DOUBLE) + .build() ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 512b7be402e9..6da277126337 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -5606,18 +5606,40 @@ public void testUnplannableExactCountDistinctOnSketch() @Test public void testArrayAggQueryOnComplexDatatypes() { + cannotVectorize(); msqCompatible(); - try { - testQuery("SELECT ARRAY_AGG(unique_dim1) FROM druid.foo", ImmutableList.of(), ImmutableList.of()); - Assert.fail("query execution should fail"); - } - catch (SqlPlanningException e) { - Assert.assertTrue( - e.getMessage().contains("Cannot use ARRAY_AGG on complex inputs COMPLEX") - ); - Assert.assertEquals(PlanningError.VALIDATION_ERROR.getErrorCode(), e.getErrorCode()); - Assert.assertEquals(PlanningError.VALIDATION_ERROR.getErrorClass(), e.getErrorClass()); - } + testQuery( + "SELECT ARRAY_AGG(unique_dim1) FROM druid.foo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("unique_dim1"), + "__acc", + "ARRAY>[]", + "ARRAY>[]", + true, + true, + false, + "array_append(\"__acc\", \"unique_dim1\")", + "array_concat(\"__acc\", \"a0\")", + null, + null, + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, + queryFramework().macroTable() + ) + )) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{"[\"AQAAAEAAAA==\",\"AQAAAQAAAAHNBA==\",\"AQAAAQAAAAOzAg==\",\"AQAAAQAAAAFREA==\",\"AQAAAQAAAACyEA==\",\"AQAAAQAAAAEkAQ==\"]"} + ) + ); } @Test From 236d9f5551e830d98431bead05410c6425c3ebfe Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 9 Feb 2023 22:40:54 -0800 Subject: [PATCH 6/7] more tests --- .../org/apache/druid/math/expr/EvalTest.java | 219 ++++++++++++++++-- 1 file changed, 197 insertions(+), 22 deletions(-) diff --git a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java index d3b5760a8042..efcda868ec9a 100644 --- a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java @@ -32,7 +32,9 @@ import org.junit.Test; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -725,6 +727,19 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.STRING, eval.type()); Assert.assertEquals("true", eval.value()); + // strings might also be liars and arrays or lists + eval = ExprEval.ofType(ExpressionType.STRING, new Object[]{"a", "b", "c"}); + Assert.assertEquals(ExpressionType.STRING_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[]{"a", "b", "c"}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.STRING, new String[]{"a", "b", "c"}); + Assert.assertEquals(ExpressionType.STRING_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[]{"a", "b", "c"}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.STRING, Arrays.asList("a", "b", "c")); + Assert.assertEquals(ExpressionType.STRING_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[]{"a", "b", "c"}, (Object[]) eval.value()); + // longs eval = ExprEval.ofType(ExpressionType.LONG, 1L); Assert.assertEquals(ExpressionType.LONG, eval.type()); @@ -778,16 +793,39 @@ public void testEvalOfType() Assert.assertEquals(type, eval.type()); Assert.assertEquals(pair, eval.value()); - // arrays fall back to using 'bestEffortOf', but cast it to the expected output type - eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {"1", "2", "3"}); + // json type isn't defined in druid-core + ExpressionType json = ExpressionType.fromString("COMPLEX"); + eval = ExprEval.ofType(json, ImmutableMap.of("x", 1L, "y", 2L)); + Assert.assertEquals(json, eval.type()); + Assert.assertEquals(ImmutableMap.of("x", 1L, "y", 2L), eval.value()); + + eval = ExprEval.ofType(json, "hello"); + Assert.assertEquals(json, eval.type()); + Assert.assertEquals("hello", eval.value()); + + ExpressionType stringyComplexThing = ExpressionType.fromString("COMPLEX"); + eval = ExprEval.ofType(stringyComplexThing, "notbase64"); + Assert.assertEquals(stringyComplexThing, eval.type()); + Assert.assertEquals("notbase64", eval.value()); + + // arrays + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1L, 2L, 3L}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); - eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {"1", "2", "wat", "3"}); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, ImmutableList.of(1L, 2L, 3L)); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); - Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); - eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1L, 2L, 3L}); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Long[]{1L, 2L, 3L}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new long[]{1L, 2L, 3L}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new int[]{1, 2, 3}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); @@ -795,10 +833,27 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); + // arrays fall might back to using 'bestEffortOf', but cast it to the expected output type + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {"1", "2", "3"}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new String[] {"1", "2", "3"}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {"1", "2", "wat", "3"}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1.0, 2.0, 3.0}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new double[] {1.0, 2.0, 3.0}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {1.0, 2.0, null, 3.0}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); @@ -807,7 +862,23 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L, 1L, 0L}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new float[] {1.0f, 2.0f, 3.0f}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + // etc + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1.0, 2.0, 3.0}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Double[] {1.0, 2.0, 3.0}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new double[] {1.0, 2.0, 3.0}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {"1", "2", "3"}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); @@ -820,17 +891,25 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new long[] {1L, 2L, 3L}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1L, 2L, null, 3L}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, NullHandling.defaultDoubleValue(), 3.0}, (Object[]) eval.value()); - eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1.0, 2.0, 3.0}); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1.0, 2L, "3", true, false}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0, 1.0, 0.0}, (Object[]) eval.value()); + + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Float[] {1.0f, 2.0f, 3.0f}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); - eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new Object[] {1.0, 2L, "3", true, false}); + eval = ExprEval.ofType(ExpressionType.DOUBLE_ARRAY, new float[] {1.0f, 2.0f, 3.0f}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); - Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0, 1.0, 0.0}, (Object[]) eval.value()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); eval = ExprEval.ofType(ExpressionType.STRING_ARRAY, new Object[] {"1", "2", "3"}); Assert.assertEquals(ExpressionType.STRING_ARRAY, eval.type()); @@ -848,20 +927,104 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.STRING_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {"1.0", "2", "3", "true", "false"}, (Object[]) eval.value()); - // json type isn't defined in druid-core - ExpressionType json = ExpressionType.fromString("COMPLEX"); - eval = ExprEval.ofType(json, ImmutableMap.of("x", 1L, "y", 2L)); - Assert.assertEquals(json, eval.type()); - Assert.assertEquals(ImmutableMap.of("x", 1L, "y", 2L), eval.value()); - - eval = ExprEval.ofType(json, "hello"); - Assert.assertEquals(json, eval.type()); - Assert.assertEquals("hello", eval.value()); - - ExpressionType stringyComplexThing = ExpressionType.fromString("COMPLEX"); - eval = ExprEval.ofType(stringyComplexThing, "notbase64"); - Assert.assertEquals(stringyComplexThing, eval.type()); - Assert.assertEquals("notbase64", eval.value()); + // nested arrays + try { + ExpressionProcessing.initializeForTests(true); + + ExpressionType nestedLongArray = ExpressionTypeFactory.getInstance().ofArray(ExpressionType.LONG_ARRAY); + final Object[] expectedLongArray = new Object[]{ + new Object[] {1L, 2L, 3L}, + new Object[] {5L, NullHandling.defaultLongValue(), 9L}, + null, + new Object[] {2L, 4L, 6L} + }; + + List longArrayInputs = Arrays.asList( + new Object[]{ + new Object[] {1L, 2L, 3L}, + new Object[] {5L, null, 9L}, + null, + new Object[] {2L, 4L, 6L} + }, + Arrays.asList( + new Object[] {1L, 2L, 3L}, + new Object[] {5L, null, 9L}, + null, + new Object[] {2L, 4L, 6L} + ), + Arrays.asList( + Arrays.asList(1L, 2L, 3L), + Arrays.asList(5L, null, 9L), + null, + Arrays.asList(2L, 4L, 6L) + ), + Arrays.asList( + Arrays.asList(1L, 2L, 3L), + Arrays.asList("5", "hello", "9"), + null, + new Object[]{2.2, 4.4, 6.6} + ) + ); + + for (Object o : longArrayInputs) { + eval = ExprEval.ofType(nestedLongArray, o); + Assert.assertEquals(nestedLongArray, eval.type()); + Object[] val = (Object[]) eval.value(); + Assert.assertEquals(expectedLongArray.length, val.length); + for (int i = 0; i < expectedLongArray.length; i++) { + Assert.assertArrayEquals((Object[]) expectedLongArray[i], (Object[]) val[i]); + } + } + + ExpressionType nestedDoubleArray = ExpressionTypeFactory.getInstance().ofArray(ExpressionType.DOUBLE_ARRAY); + final Object[] expectedDoubleArray = new Object[]{ + new Object[] {1.1, 2.2, 3.3}, + new Object[] {5.5, NullHandling.defaultDoubleValue(), 9.9}, + null, + new Object[] {2.2, 4.4, 6.6} + }; + + List doubleArrayInputs = Arrays.asList( + new Object[]{ + new Object[] {1.1, 2.2, 3.3}, + new Object[] {5.5, null, 9.9}, + null, + new Object[] {2.2, 4.4, 6.6} + }, + new Object[]{ + Arrays.asList(1.1, 2.2, 3.3), + Arrays.asList(5.5, null, 9.9), + null, + Arrays.asList(2.2, 4.4, 6.6) + }, + Arrays.asList( + Arrays.asList(1.1, 2.2, 3.3), + Arrays.asList(5.5, null, 9.9), + null, + Arrays.asList(2.2, 4.4, 6.6) + ), + new Object[]{ + new Object[] {"1.1", "2.2", "3.3"}, + Arrays.asList("5.5", null, "9.9"), + null, + new String[] {"2.2", "4.4", "6.6"} + } + ); + + for (Object o : doubleArrayInputs) { + eval = ExprEval.ofType(nestedDoubleArray, o); + Assert.assertEquals(nestedDoubleArray, eval.type()); + Object[] val = (Object[]) eval.value(); + Assert.assertEquals(expectedLongArray.length, val.length); + for (int i = 0; i < expectedLongArray.length; i++) { + Assert.assertArrayEquals((Object[]) expectedDoubleArray[i], (Object[]) val[i]); + } + } + } + finally { + // reset + ExpressionProcessing.initializeForTests(null); + } } @Test @@ -932,6 +1095,10 @@ public void testBestEffortOf() Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.bestEffortOf(new Integer[] {1, 2, 3}); + Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); + eval = ExprEval.bestEffortOf(new int[] {1, 2, 3}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); @@ -944,6 +1111,10 @@ public void testBestEffortOf() Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {NullHandling.defaultDoubleValue(), 1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.bestEffortOf(new Double[] {1.0, 2.0, 3.0}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.bestEffortOf(new double[] {1.0, 2.0, 3.0}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); @@ -952,6 +1123,10 @@ public void testBestEffortOf() Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.bestEffortOf(new Float[] {1.0f, 2.0f, 3.0f}); + Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); + Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); + eval = ExprEval.bestEffortOf(new float[] {1.0f, 2.0f, 3.0f}); Assert.assertEquals(ExpressionType.DOUBLE_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1.0, 2.0, 3.0}, (Object[]) eval.value()); From fcba37ab80cc669b185b05de9d5f6cd238961ca5 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 10 Feb 2023 16:20:55 -0800 Subject: [PATCH 7/7] cleanup --- core/src/test/java/org/apache/druid/math/expr/EvalTest.java | 2 +- .../apache/druid/sql/calcite/CalciteNestedDataQueryTest.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java index efcda868ec9a..c3322ba19f7a 100644 --- a/core/src/test/java/org/apache/druid/math/expr/EvalTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/EvalTest.java @@ -833,7 +833,7 @@ public void testEvalOfType() Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, NullHandling.defaultLongValue(), 3L}, (Object[]) eval.value()); - // arrays fall might back to using 'bestEffortOf', but cast it to the expected output type + // arrays might have to fall back to using 'bestEffortOf', but will cast it to the expected output type eval = ExprEval.ofType(ExpressionType.LONG_ARRAY, new Object[] {"1", "2", "3"}); Assert.assertEquals(ExpressionType.LONG_ARRAY, eval.type()); Assert.assertArrayEquals(new Object[] {1L, 2L, 3L}, (Object[]) eval.value()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 840c1f963065..36464032e368 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -881,9 +881,6 @@ public void testJsonAndArrayAgg() ) .setAggregatorSpecs( aggregators( - /* - combineExpressionString='array_concat("__acc", "a0")', compareExpressionString='null', finalizeExpressionString='null', maxSizeBytes=16384} - */ new ExpressionLambdaAggregatorFactory( "a0", ImmutableSet.of("nest"),