From 4a942cebaa27b5e2b625ad0d03b1d24cef9293b0 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 12 May 2021 04:43:16 -0700 Subject: [PATCH 1/6] add string_agg --- docs/querying/sql.md | 8 +- .../ExpressionLambdaAggregatorFactory.java | 2 +- .../builtin/ArraySqlAggregator.java | 4 +- .../builtin/StringSqlAggregator.java | 199 +++++++++ .../calcite/planner/DruidOperatorTable.java | 28 +- .../druid/sql/calcite/CalciteQueryTest.java | 422 +++++++++++++++++- 6 files changed, 638 insertions(+), 25 deletions(-) create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java diff --git a/docs/querying/sql.md b/docs/querying/sql.md index 1fbed13b96de..dab61f9d6447 100644 --- a/docs/querying/sql.md +++ b/docs/querying/sql.md @@ -316,7 +316,7 @@ no matches while aggregating values across an entire table without a grouping, o within a grouping. What this value is exactly varies per aggregator, but COUNT, and the various approximate count distinct sketch functions, will always return 0. -Only the COUNT and ARRAY_AGG aggregations can accept the DISTINCT keyword. +Only the COUNT, ARRAY_AGG, and STRING_AGG aggregations can accept the DISTINCT keyword. > The order of aggregation operations across segments is not deterministic. This means that non-commutative aggregation > functions can produce inconsistent results across the same query. @@ -358,8 +358,10 @@ Only the COUNT and ARRAY_AGG aggregations can accept the DISTINCT keyword. |`ANY_VALUE(expr)`|Returns any value of `expr` including null. `expr` must be numeric. This aggregator can simplify and optimize the performance by returning the first encountered value (including null)|`null` if `druid.generic.useDefaultValueForNull=false`, otherwise `0`| |`ANY_VALUE(expr, maxBytesPerString)`|Like `ANY_VALUE(expr)`, but for strings. The `maxBytesPerString` parameter determines how much aggregation space to allocate per string. Strings longer than this limit will be truncated. This parameter should be set as low as possible, since high values will lead to wasted memory.|`null` if `druid.generic.useDefaultValueForNull=false`, otherwise `''`| |`GROUPING(expr, expr...)`|Returns a number to indicate which groupBy dimension is included in a row, when using `GROUPING SETS`. Refer to [additional documentation](aggregations.md#grouping-aggregator) on how to infer this number.|N/A| -|`ARRAY_AGG(expr, [size])`|Collects all values of `expr` into an ARRAY, including null values, with `size` in bytes limit on aggregation size (default of 1024 bytes). Use of `ORDER BY` within the `ARRAY_AGG` expression is not currently supported, and the ordering of results within the output array may vary depending on processing order.|`null`| -|`ARRAY_AGG(DISTINCT expr, [size])`|Collects all distinct values of `expr` into an ARRAY, including null values, with `size` in bytes limit on aggregation size (default of 1024 bytes) per aggregate. Use of `ORDER BY` within the `ARRAY_AGG` expression is not currently supported, and the ordering of results within the output array may vary depending on processing order.|`null`| +|`ARRAY_AGG(expr, [size])`|Collects all values of `expr` into an ARRAY, including null values, with `size` in bytes limit on aggregation size (default of 1024 bytes). If the aggregated array grows larger than the maximum size in bytes, the query will fail. Use of `ORDER BY` within the `ARRAY_AGG` expression is not currently supported, and the ordering of results within the output array may vary depending on processing order.|`null`| +|`ARRAY_AGG(DISTINCT expr, [size])`|Collects all distinct values of `expr` into an ARRAY, including null values, with `size` in bytes limit on aggregation size (default of 1024 bytes) per aggregate. If the aggregated array grows larger than the maximum size in bytes, the query will fail. Use of `ORDER BY` within the `ARRAY_AGG` expression is not currently supported, and the ordering of results within the output array may vary depending on processing order.|`null`| +|`STRING_AGG(expr, separator, [size])`|Collects all values of `expr` into a single STRING, ignoring null values. Each value is joined by the `separator` which must be a literal STRING. An optional `size` in bytes can be supplied to limit aggregation size (default of 1024 bytes). If the aggregated string grows larger than the maximum size in bytes, the query will fail. Use of `ORDER BY` within the `STRING_AGG` expression is not currently supported, and the ordering of results within the output string may vary depending on processing order.|`null` if `druid.generic.useDefaultValueForNull=false`, otherwise `''`| +|`STRING_AGG(DISTINCT expr, separator, [size])`|Collects all distinct values of `expr` into a single STRING, ignoring null values. Each value is joined by the `separator` which must be a literal STRING. An optional `size` in bytes can be supplied to limit aggregation size (default of 1024 bytes). If the aggregated string grows larger than the maximum size in bytes, the query will fail. Use of `ORDER BY` within the `STRING_AGG` expression is not currently supported, and the ordering of results within the output string may vary depending on processing order.|`null` if `druid.generic.useDefaultValueForNull=false`, otherwise `''`| For advice on choosing approximate aggregation functions, check out our [approximate aggregations documentation](aggregations.md#approx). diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/ExpressionLambdaAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/ExpressionLambdaAggregatorFactory.java index be8b1004e55b..7a85fa992562 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/ExpressionLambdaAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/ExpressionLambdaAggregatorFactory.java @@ -65,7 +65,7 @@ public class ExpressionLambdaAggregatorFactory extends AggregatorFactory // minimum permitted agg size is 10 bytes so it is at least large enough to hold primitive numerics (long, double) // | expression type byte | is_null byte | primitive value (8 bytes) | private static final int MIN_SIZE_BYTES = 10; - private static final HumanReadableBytes DEFAULT_MAX_SIZE_BYTES = new HumanReadableBytes(1L << 10); + public static final HumanReadableBytes DEFAULT_MAX_SIZE_BYTES = new HumanReadableBytes(1L << 10); private final String name; @Nullable diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java index 0f80daa91fa0..78ea70a5f2ab 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ArraySqlAggregator.java @@ -182,7 +182,7 @@ public RelDataType inferReturnType(SqlOperatorBinding sqlOperatorBinding) { RelDataType type = sqlOperatorBinding.getOperandType(0); if (SqlTypeUtil.isArray(type)) { - throw new ISE("Cannot ARRAY_AGG on array inputs %s", type); + throw new ISE("Cannot use ARRAY_AGG on array inputs %s", type); } return Calcites.createSqlArrayTypeWithNullability( sqlOperatorBinding.getTypeFactory(), @@ -207,7 +207,7 @@ private static class ArrayAggFunction extends SqlAggFunction OperandTypes.or( OperandTypes.ANY, OperandTypes.and( - OperandTypes.sequence(StringUtils.format("'%s'(expr, maxSizeBytes)", NAME), OperandTypes.ANY, OperandTypes.LITERAL), + OperandTypes.sequence(StringUtils.format("'%s'(expr, maxSizeBytes)", NAME), OperandTypes.ANY, OperandTypes.POSITIVE_INTEGER_LITERAL), OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.NUMERIC) ) ), diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java new file mode 100644 index 000000000000..6302d130467d --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java @@ -0,0 +1,199 @@ +package org.apache.druid.sql.calcite.aggregation.builtin; + +import com.google.common.collect.ImmutableSet; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.type.InferTypes; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlTypeFamily; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.Optionality; +import org.apache.druid.java.util.common.HumanReadableBytes; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.aggregation.ExpressionLambdaAggregatorFactory; +import org.apache.druid.query.aggregation.FilteredAggregatorFactory; +import org.apache.druid.query.filter.NotDimFilter; +import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.sql.calcite.aggregation.Aggregation; +import org.apache.druid.sql.calcite.aggregation.SqlAggregator; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.Expressions; +import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class StringSqlAggregator implements SqlAggregator +{ + private static final String NAME = "STRING_AGG"; + private static final SqlAggFunction FUNCTION = new StringAggFunction(); + @Override + public SqlAggFunction calciteFunction() + { + return FUNCTION; + } + + @Nullable + @Override + public Aggregation toDruidAggregation( + PlannerContext plannerContext, + RowSignature rowSignature, + VirtualColumnRegistry virtualColumnRegistry, + RexBuilder rexBuilder, + String name, + AggregateCall aggregateCall, + Project project, + List existingAggregations, + boolean finalizeAggregations + ) + { + final List arguments = aggregateCall + .getArgList() + .stream() + .map(i -> Expressions.fromFieldAccess(rowSignature, project, i)) + .map(rexNode -> Expressions.toDruidExpression(plannerContext, rowSignature, rexNode)) + .collect(Collectors.toList()); + + if (arguments.stream().anyMatch(Objects::isNull)) { + return null; + } + + RexNode separatorNode = Expressions.fromFieldAccess( + rowSignature, + project, + aggregateCall.getArgList().get(1) + ); + if (!separatorNode.isA(SqlKind.LITERAL)) { + // separator must be a literal + return null; + } + String separator = RexLiteral.stringValue(separatorNode); + + if (separator == null) { + // separator must not be null + return null; + } + + Integer maxSizeBytes = null; + if (arguments.size() > 2) { + RexNode maxBytes = Expressions.fromFieldAccess( + rowSignature, + project, + aggregateCall.getArgList().get(2) + ); + if (!maxBytes.isA(SqlKind.LITERAL)) { + // maxBytes must be a literal + return null; + } + maxSizeBytes = ((Number) RexLiteral.value(maxBytes)).intValue(); + } + final DruidExpression arg = arguments.get(0); + final ExprMacroTable macroTable = plannerContext.getExprMacroTable(); + + final String initialvalue = "[]"; + final ValueType elementType = ValueType.STRING; + final String fieldName; + if (arg.isDirectColumnAccess()) { + fieldName = arg.getDirectColumn(); + } else { + VirtualColumn vc = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, arg, elementType); + fieldName = vc.getOutputName(); + } + + final String finalizer = StringUtils.format("if(array_length(o) == 0, null, array_to_string(o, '%s'))", separator); + final NotDimFilter dimFilter = new NotDimFilter(new SelectorDimFilter(fieldName, null, null)); + if (aggregateCall.isDistinct()) { + return Aggregation.create( + // string_agg ignores nulls + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + name, + ImmutableSet.of(fieldName), + null, + initialvalue, + null, + StringUtils.format("array_set_add(\"__acc\", \"%s\")", fieldName), + StringUtils.format("array_set_add_all(\"__acc\", \"%s\")", name), + null, + finalizer, + maxSizeBytes != null ? new HumanReadableBytes(maxSizeBytes) : null, + macroTable + ), + dimFilter + ) + ); + } else { + return Aggregation.create( + // string_agg ignores nulls + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + name, + ImmutableSet.of(fieldName), + null, + initialvalue, + null, + StringUtils.format("array_append(\"__acc\", \"%s\")", fieldName), + StringUtils.format("array_concat(\"__acc\", \"%s\")", name), + null, + finalizer, + maxSizeBytes != null ? new HumanReadableBytes(maxSizeBytes) : null, + macroTable + ), + dimFilter + ) + ); + } + } + + private static class StringAggFunction extends SqlAggFunction + { + StringAggFunction() + { + super( + NAME, + null, + SqlKind.OTHER_FUNCTION, + opBinding -> + Calcites.createSqlTypeWithNullability(opBinding.getTypeFactory(), SqlTypeName.VARCHAR, true), + InferTypes.ANY_NULLABLE, + OperandTypes.or( + OperandTypes.and( + OperandTypes.sequence( + StringUtils.format("'%s'(expr, separator)", NAME), + OperandTypes.ANY, + OperandTypes.LITERAL + ), + OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.STRING) + ), + OperandTypes.and( + OperandTypes.sequence( + StringUtils.format("'%s'(expr, separator, maxSizeBytes)", NAME), + OperandTypes.ANY, + OperandTypes.LITERAL, + OperandTypes.POSITIVE_INTEGER_LITERAL + ), + OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.STRING, SqlTypeFamily.NUMERIC) + ) + ), + SqlFunctionCategory.STRING, + false, + false, + Optionality.IGNORED + ); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java index f0135b7c0a5a..ea6b27c823cc 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidOperatorTable.java @@ -41,6 +41,7 @@ import org.apache.druid.sql.calcite.aggregation.builtin.GroupingSqlAggregator; import org.apache.druid.sql.calcite.aggregation.builtin.MaxSqlAggregator; import org.apache.druid.sql.calcite.aggregation.builtin.MinSqlAggregator; +import org.apache.druid.sql.calcite.aggregation.builtin.StringSqlAggregator; import org.apache.druid.sql.calcite.aggregation.builtin.SumSqlAggregator; import org.apache.druid.sql.calcite.aggregation.builtin.SumZeroSqlAggregator; import org.apache.druid.sql.calcite.expression.AliasedOperatorConversion; @@ -123,19 +124,20 @@ public class DruidOperatorTable implements SqlOperatorTable { private static final List STANDARD_AGGREGATORS = ImmutableList.builder() - .add(new ApproxCountDistinctSqlAggregator()) - .add(new AvgSqlAggregator()) - .add(new CountSqlAggregator()) - .add(EarliestLatestAnySqlAggregator.EARLIEST) - .add(EarliestLatestAnySqlAggregator.LATEST) - .add(EarliestLatestAnySqlAggregator.ANY_VALUE) - .add(new MinSqlAggregator()) - .add(new MaxSqlAggregator()) - .add(new SumSqlAggregator()) - .add(new SumZeroSqlAggregator()) - .add(new GroupingSqlAggregator()) - .add(new ArraySqlAggregator()) - .build(); + .add(new ApproxCountDistinctSqlAggregator()) + .add(new AvgSqlAggregator()) + .add(new CountSqlAggregator()) + .add(EarliestLatestAnySqlAggregator.EARLIEST) + .add(EarliestLatestAnySqlAggregator.LATEST) + .add(EarliestLatestAnySqlAggregator.ANY_VALUE) + .add(new MinSqlAggregator()) + .add(new MaxSqlAggregator()) + .add(new SumSqlAggregator()) + .add(new SumZeroSqlAggregator()) + .add(new GroupingSqlAggregator()) + .add(new ArraySqlAggregator()) + .add(new StringSqlAggregator()) + .build(); // STRLEN has so many aliases. diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 976d61a0e202..b45a6ad79a3f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -12619,7 +12619,8 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr + " EARLIEST(l1),\n" + " LATEST(dim1, 1024),\n" + " LATEST(l1),\n" - + " ARRAY_AGG(DISTINCT dim3)\n" + + " ARRAY_AGG(DISTINCT dim3),\n" + + " STRING_AGG(DISTINCT dim3, '|')\n" + "FROM druid.numfoo WHERE dim2 = 0", ImmutableList.of( Druids.newTimeseriesQueryBuilder() @@ -12647,6 +12648,22 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "if(array_length(o) == 0, null, o)", new HumanReadableBytes(1024), TestExprMacroTable.INSTANCE + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a7", + ImmutableSet.of("dim3"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim3\")", + "array_set_add_all(\"__acc\", \"a7\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, '|'))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("dim3", null, null)) ) ) ) @@ -12655,8 +12672,8 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr ), ImmutableList.of( useDefault - ? new Object[]{"", 0L, "", 0L, "", 0L, null} - : new Object[]{null, null, null, null, null, null, null} + ? new Object[]{"", 0L, "", 0L, "", 0L, null, ""} + : new Object[]{null, null, null, null, null, null, null, null} ) ); } @@ -12797,7 +12814,8 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception + " EARLIEST(l1) FILTER(WHERE dim1 = 'nonexistent'),\n" + " LATEST(dim1, 1024) FILTER(WHERE dim1 = 'nonexistent'),\n" + " LATEST(l1) FILTER(WHERE dim1 = 'nonexistent'),\n" - + " ARRAY_AGG(DISTINCT dim3) FILTER(WHERE dim1 = 'nonexistent')" + + " ARRAY_AGG(DISTINCT dim3) FILTER(WHERE dim1 = 'nonexistent'),\n" + + " STRING_AGG(DISTINCT dim3, '|') FILTER(WHERE dim1 = 'nonexistent')\n" + "FROM druid.numfoo WHERE dim2 = 'a' GROUP BY dim2", ImmutableList.of( GroupByQuery.builder() @@ -12848,6 +12866,25 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception TestExprMacroTable.INSTANCE ), selector("dim1", "nonexistent", null) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a7", + ImmutableSet.of("dim3"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim3\")", + "array_set_add_all(\"__acc\", \"a7\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, '|'))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + and( + selector("dim1", "nonexistent", null), + not(selector("dim3", null, null)) + ) ) ) ) @@ -12856,8 +12893,8 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception ), ImmutableList.of( useDefault - ? new Object[]{"a", "", 0L, "", 0L, "", 0L, null} - : new Object[]{"a", null, null, null, null, null, null, null} + ? new Object[]{"a", "", 0L, "", 0L, "", 0L, null, ""} + : new Object[]{"a", null, null, null, null, null, null, null, null} ) ); } @@ -19001,4 +19038,377 @@ public void testCountAndAverageByConstantVirtualColumn() throws Exception ) ); } + + @Test + public void testStringAgg() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT STRING_AGG(dim1,','), STRING_AGG(DISTINCT dim1, ','), STRING_AGG(DISTINCT dim1,',') FILTER(WHERE dim1 = 'shazbot') FROM foo WHERE dim1 is not null", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(not(selector("dim1", null, null))) + .aggregators( + aggregators( + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"dim1\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("dim1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("dim1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a2", + ImmutableSet.of("dim1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim1\")", + "array_set_add_all(\"__acc\", \"a2\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + and( + not(selector("dim1", null, null)), + selector("dim1", "shazbot", null) + ) + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{"10.1,2,1,def,abc", "1,2,abc,def,10.1", ""} + : new Object[]{",10.1,2,1,def,abc", ",1,2,abc,def,10.1", null} + ) + ); + } + + @Test + public void testStringAggMultiValue() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT STRING_AGG(dim3, ','), STRING_AGG(DISTINCT dim3, ',') FROM foo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators( + aggregators( + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("dim3"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"dim3\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("dim3", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("dim3"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"dim3\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("dim3", null, null)) + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{"a,b,b,c,d", "a,b,c,d"} + : new Object[]{"a,b,b,c,d,", ",a,b,c,d"} + ) + ); + } + + @Test + public void testStringAggNumeric() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT STRING_AGG(l1, ','), STRING_AGG(DISTINCT l1, ','), STRING_AGG(d1, ','), STRING_AGG(DISTINCT d1, ','), STRING_AGG(f1, ','), STRING_AGG(DISTINCT f1, ',') FROM numfoo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators( + aggregators( + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"l1\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("l1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"l1\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("l1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a2", + ImmutableSet.of("d1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"d1\")", + "array_concat(\"__acc\", \"a2\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("d1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a3", + ImmutableSet.of("d1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"d1\")", + "array_set_add_all(\"__acc\", \"a3\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("d1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a4", + ImmutableSet.of("f1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"f1\")", + "array_concat(\"__acc\", \"a4\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("f1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a5", + ImmutableSet.of("f1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"f1\")", + "array_set_add_all(\"__acc\", \"a5\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("f1", null, null)) + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{ + "7,325323,0,0,0,0", + "0,7,325323", + "1.0,1.7,0.0,0.0,0.0,0.0", + "0.0,1.0,1.7", + "1.0,0.10000000149011612,0.0,0.0,0.0,0.0", + "0.10000000149011612,0.0,1.0" + } + : new Object[]{ + "7,325323,0", + "0,7,325323", + "1.0,1.7,0.0", + "0.0,1.0,1.7", + "1.0,0.10000000149011612,0.0", + "0.10000000149011612,0.0,1.0" + } + ) + ); + } + + @Test + public void testStringAggExpression() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT STRING_AGG(DISTINCT CONCAT(dim1, dim2), ',') FROM foo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .virtualColumns( + expressionVirtualColumn("v0", "concat(\"dim1\",\"dim2\")", ValueType.STRING) + ) + .aggregators( + aggregators( + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("v0"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"v0\")", + "array_set_add_all(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(1024), + TestExprMacroTable.INSTANCE + ), + not(selector("v0", null, null)) + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault ? new Object[]{"1a,a,2,abc,10.1,defabc"} : new Object[]{"1a,a,2,defabc"} + ) + ); + } + + @Test + public void testStringAggMaxBytes() throws Exception + { + cannotVectorize(); + testQuery( + "SELECT STRING_AGG(l1, ',', 128), STRING_AGG(DISTINCT l1, ',', 128) FROM numfoo", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators( + aggregators( + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a0", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_append(\"__acc\", \"l1\")", + "array_concat(\"__acc\", \"a0\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(128), + TestExprMacroTable.INSTANCE + ), + not(selector("l1", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("l1"), + "__acc", + "[]", + "[]", + "array_set_add(\"__acc\", \"l1\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, ','))", + new HumanReadableBytes(128), + TestExprMacroTable.INSTANCE + ), + not(selector("l1", null, null)) + ) + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + useDefault + ? new Object[]{"7,325323,0,0,0,0", "0,7,325323"} + : new Object[]{"7,325323,0", "0,7,325323"} + ) + ); + } } From f0b7063208d29e46261323cd66cc2d97ac403efa Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 12 May 2021 04:56:52 -0700 Subject: [PATCH 2/6] oops --- .../builtin/StringSqlAggregator.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java index 6302d130467d..9dc3c12367e5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.aggregation.builtin; import com.google.common.collect.ImmutableSet; From 7035d97c36738bb2582be0b0d10f751a616f1382 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 May 2021 00:21:03 -0700 Subject: [PATCH 3/6] style and fix test --- .../sql/calcite/aggregation/builtin/StringSqlAggregator.java | 1 + .../java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java index 9dc3c12367e5..f72fd9e43ff8 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java @@ -60,6 +60,7 @@ public class StringSqlAggregator implements SqlAggregator { private static final String NAME = "STRING_AGG"; private static final SqlAggFunction FUNCTION = new StringAggFunction(); + @Override public SqlAggFunction calciteFunction() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index b45a6ad79a3f..38f53cbeca38 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -12882,8 +12882,8 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception TestExprMacroTable.INSTANCE ), and( - selector("dim1", "nonexistent", null), - not(selector("dim3", null, null)) + not(selector("dim3", null, null)), + selector("dim1", "nonexistent", null) ) ) ) From 314f795ff23aea2fd51eee4902874d8e63aaeb27 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 13 May 2021 02:09:58 -0700 Subject: [PATCH 4/6] spelling --- website/.spelling | 1 + 1 file changed, 1 insertion(+) diff --git a/website/.spelling b/website/.spelling index 1219f567e677..6edda355aff1 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1530,6 +1530,7 @@ SCHEMA_OWNER SERVER_SEGMENTS SMALLINT SQL_PATH +STRING_AGG SYSTEM_TABLE TABLE_CATALOG TABLE_NAME From 2770e0ea93f7619c8e089f6bcb61003410afa9f1 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 28 Jun 2021 21:41:20 -0700 Subject: [PATCH 5/6] fixup --- .../builtin/StringSqlAggregator.java | 2 ++ .../druid/sql/calcite/CalciteQueryTest.java | 30 ++++++++++++++----- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java index f72fd9e43ff8..f8d6e18e1a61 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java @@ -146,6 +146,7 @@ public Aggregation toDruidAggregation( null, initialvalue, null, + true, StringUtils.format("array_set_add(\"__acc\", \"%s\")", fieldName), StringUtils.format("array_set_add_all(\"__acc\", \"%s\")", name), null, @@ -166,6 +167,7 @@ public Aggregation toDruidAggregation( null, initialvalue, null, + true, StringUtils.format("array_append(\"__acc\", \"%s\")", fieldName), StringUtils.format("array_concat(\"__acc\", \"%s\")", name), null, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 898f6c22e98f..3570f026fe06 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -12652,7 +12652,6 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr + " LATEST(l1),\n" + " ARRAY_AGG(DISTINCT dim3),\n" + " STRING_AGG(DISTINCT dim3, '|'),\n" - + " ARRAY_AGG(DISTINCT dim3),\n" + " BIT_AND(l1),\n" + " BIT_OR(l1),\n" + " BIT_XOR(l1)\n" @@ -12692,6 +12691,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"dim3\")", "array_set_add_all(\"__acc\", \"a7\")", null, @@ -12710,7 +12710,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "0", NullHandling.sqlCompatible(), "bitwiseAnd(\"__acc\", \"l1\")", - "bitwiseAnd(\"__acc\", \"a7\")", + "bitwiseAnd(\"__acc\", \"a8\")", null, null, new HumanReadableBytes(1024), @@ -12727,7 +12727,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "0", NullHandling.sqlCompatible(), "bitwiseOr(\"__acc\", \"l1\")", - "bitwiseOr(\"__acc\", \"a8\")", + "bitwiseOr(\"__acc\", \"a9\")", null, null, new HumanReadableBytes(1024), @@ -12744,7 +12744,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "0", NullHandling.sqlCompatible(), "bitwiseXor(\"__acc\", \"l1\")", - "bitwiseXor(\"__acc\", \"a9\")", + "bitwiseXor(\"__acc\", \"a10\")", null, null, new HumanReadableBytes(1024), @@ -12903,7 +12903,6 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception + " LATEST(l1) FILTER(WHERE dim1 = 'nonexistent'),\n" + " ARRAY_AGG(DISTINCT dim3) FILTER(WHERE dim1 = 'nonexistent'),\n" + " STRING_AGG(DISTINCT dim3, '|') FILTER(WHERE dim1 = 'nonexistent'),\n" - + " ARRAY_AGG(DISTINCT dim3) FILTER(WHERE dim1 = 'nonexistent'),\n" + " BIT_AND(l1) FILTER(WHERE dim1 = 'nonexistent'),\n" + " BIT_OR(l1) FILTER(WHERE dim1 = 'nonexistent'),\n" + " BIT_XOR(l1) FILTER(WHERE dim1 = 'nonexistent')\n" @@ -12966,6 +12965,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"dim3\")", "array_set_add_all(\"__acc\", \"a7\")", null, @@ -12987,7 +12987,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "0", NullHandling.sqlCompatible(), "bitwiseAnd(\"__acc\", \"l1\")", - "bitwiseAnd(\"__acc\", \"a7\")", + "bitwiseAnd(\"__acc\", \"a8\")", null, null, new HumanReadableBytes(1024), @@ -13004,7 +13004,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "0", NullHandling.sqlCompatible(), "bitwiseOr(\"__acc\", \"l1\")", - "bitwiseOr(\"__acc\", \"a8\")", + "bitwiseOr(\"__acc\", \"a9\")", null, null, new HumanReadableBytes(1024), @@ -13021,7 +13021,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "0", NullHandling.sqlCompatible(), "bitwiseXor(\"__acc\", \"l1\")", - "bitwiseXor(\"__acc\", \"a9\")", + "bitwiseXor(\"__acc\", \"a10\")", null, null, new HumanReadableBytes(1024), @@ -17964,6 +17964,7 @@ public void testStringAgg() throws Exception "__acc", "[]", "[]", + true, "array_append(\"__acc\", \"dim1\")", "array_concat(\"__acc\", \"a0\")", null, @@ -17980,6 +17981,7 @@ public void testStringAgg() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"dim1\")", "array_set_add_all(\"__acc\", \"a1\")", null, @@ -17996,6 +17998,7 @@ public void testStringAgg() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"dim1\")", "array_set_add_all(\"__acc\", \"a2\")", null, @@ -18041,6 +18044,7 @@ public void testStringAggMultiValue() throws Exception "__acc", "[]", "[]", + true, "array_append(\"__acc\", \"dim3\")", "array_concat(\"__acc\", \"a0\")", null, @@ -18057,6 +18061,7 @@ public void testStringAggMultiValue() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"dim3\")", "array_set_add_all(\"__acc\", \"a1\")", null, @@ -18099,6 +18104,7 @@ public void testStringAggNumeric() throws Exception "__acc", "[]", "[]", + true, "array_append(\"__acc\", \"l1\")", "array_concat(\"__acc\", \"a0\")", null, @@ -18115,6 +18121,7 @@ public void testStringAggNumeric() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"l1\")", "array_set_add_all(\"__acc\", \"a1\")", null, @@ -18131,6 +18138,7 @@ public void testStringAggNumeric() throws Exception "__acc", "[]", "[]", + true, "array_append(\"__acc\", \"d1\")", "array_concat(\"__acc\", \"a2\")", null, @@ -18147,6 +18155,7 @@ public void testStringAggNumeric() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"d1\")", "array_set_add_all(\"__acc\", \"a3\")", null, @@ -18163,6 +18172,7 @@ public void testStringAggNumeric() throws Exception "__acc", "[]", "[]", + true, "array_append(\"__acc\", \"f1\")", "array_concat(\"__acc\", \"a4\")", null, @@ -18179,6 +18189,7 @@ public void testStringAggNumeric() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"f1\")", "array_set_add_all(\"__acc\", \"a5\")", null, @@ -18238,6 +18249,7 @@ public void testStringAggExpression() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"v0\")", "array_set_add_all(\"__acc\", \"a0\")", null, @@ -18278,6 +18290,7 @@ public void testStringAggMaxBytes() throws Exception "__acc", "[]", "[]", + true, "array_append(\"__acc\", \"l1\")", "array_concat(\"__acc\", \"a0\")", null, @@ -18294,6 +18307,7 @@ public void testStringAggMaxBytes() throws Exception "__acc", "[]", "[]", + true, "array_set_add(\"__acc\", \"l1\")", "array_set_add_all(\"__acc\", \"a1\")", null, From 4e8b0179657cf74534018329302b78f041d84782 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 28 Jun 2021 22:04:06 -0700 Subject: [PATCH 6/6] review stuffs --- .../builtin/StringSqlAggregator.java | 4 +- .../sql/calcite/CalciteArraysQueryTest.java | 32 +++---- .../druid/sql/calcite/CalciteQueryTest.java | 87 ++++++++++++------- 3 files changed, 75 insertions(+), 48 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java index f8d6e18e1a61..596f69b13299 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/StringSqlAggregator.java @@ -197,7 +197,7 @@ private static class StringAggFunction extends SqlAggFunction OperandTypes.sequence( StringUtils.format("'%s'(expr, separator)", NAME), OperandTypes.ANY, - OperandTypes.LITERAL + OperandTypes.STRING ), OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.STRING) ), @@ -205,7 +205,7 @@ private static class StringAggFunction extends SqlAggFunction OperandTypes.sequence( StringUtils.format("'%s'(expr, separator, maxSizeBytes)", NAME), OperandTypes.ANY, - OperandTypes.LITERAL, + OperandTypes.STRING, OperandTypes.POSITIVE_INTEGER_LITERAL ), OperandTypes.family(SqlTypeFamily.ANY, SqlTypeFamily.STRING, SqlTypeFamily.NUMERIC) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 00aebaf2b1ca..78382b260029 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -1161,7 +1161,7 @@ public void testArrayAgg() throws Exception "array_concat(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1175,7 +1175,7 @@ public void testArrayAgg() throws Exception "array_set_add_all(\"__acc\", \"a1\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new FilteredAggregatorFactory( @@ -1190,7 +1190,7 @@ public void testArrayAgg() throws Exception "array_set_add_all(\"__acc\", \"a2\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), selector("dim1", "shazbot", null) @@ -1236,7 +1236,7 @@ public void testArrayAggMultiValue() throws Exception "array_concat(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1250,7 +1250,7 @@ public void testArrayAggMultiValue() throws Exception "array_set_add_all(\"__acc\", \"a1\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) @@ -1290,7 +1290,7 @@ public void testArrayAggNumeric() throws Exception "array_concat(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1304,7 +1304,7 @@ public void testArrayAggNumeric() throws Exception "array_set_add_all(\"__acc\", \"a1\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1318,7 +1318,7 @@ public void testArrayAggNumeric() throws Exception "array_concat(\"__acc\", \"a2\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1332,7 +1332,7 @@ public void testArrayAggNumeric() throws Exception "array_set_add_all(\"__acc\", \"a3\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1346,7 +1346,7 @@ public void testArrayAggNumeric() throws Exception "array_concat(\"__acc\", \"a4\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new ExpressionLambdaAggregatorFactory( @@ -1360,7 +1360,7 @@ public void testArrayAggNumeric() throws Exception "array_set_add_all(\"__acc\", \"a5\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) @@ -1415,7 +1415,7 @@ public void testArrayAggToString() throws Exception "array_set_add_all(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) @@ -1457,7 +1457,7 @@ public void testArrayAggExpression() throws Exception "array_set_add_all(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) @@ -1577,7 +1577,7 @@ public void testArrayAggAsArrayFromJoin() throws Exception "array_set_add_all(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) @@ -1669,7 +1669,7 @@ public void testArrayAggArrayContainsSubquery() throws Exception "array_set_add_all(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) @@ -1746,7 +1746,7 @@ public void testArrayAggGroupByArrayContainsSubquery() throws Exception "array_set_add_all(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ) ) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 3570f026fe06..a49bf6bc343c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -12681,7 +12681,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "array_set_add_all(\"__acc\", \"a6\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), new FilteredAggregatorFactory( @@ -12696,7 +12696,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "array_set_add_all(\"__acc\", \"a7\")", null, "if(array_length(o) == 0, null, array_to_string(o, '|'))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("dim3", null, null)) @@ -12713,7 +12713,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "bitwiseAnd(\"__acc\", \"a8\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -12730,7 +12730,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "bitwiseOr(\"__acc\", \"a9\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -12747,7 +12747,7 @@ public void testTimeseriesEmptyResultsAggregatorDefaultValuesNonVectorized() thr "bitwiseXor(\"__acc\", \"a10\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -12953,7 +12953,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "array_set_add_all(\"__acc\", \"a6\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), selector("dim1", "nonexistent", null) @@ -12970,7 +12970,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "array_set_add_all(\"__acc\", \"a7\")", null, "if(array_length(o) == 0, null, array_to_string(o, '|'))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), and( @@ -12990,7 +12990,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "bitwiseAnd(\"__acc\", \"a8\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), and(not(selector("l1", null, null)), selector("dim1", "nonexistent", null)) @@ -13007,7 +13007,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "bitwiseOr(\"__acc\", \"a9\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), and(not(selector("l1", null, null)), selector("dim1", "nonexistent", null)) @@ -13024,7 +13024,7 @@ public void testGroupByAggregatorDefaultValuesNonVectorized() throws Exception "bitwiseXor(\"__acc\", \"a10\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), and(not(selector("l1", null, null)), selector("dim1", "nonexistent", null)) @@ -17790,7 +17790,7 @@ public void testBitwiseAggregatorsTimeseries() throws Exception "bitwiseAnd(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -17807,7 +17807,7 @@ public void testBitwiseAggregatorsTimeseries() throws Exception "bitwiseOr(\"__acc\", \"a1\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -17824,7 +17824,7 @@ public void testBitwiseAggregatorsTimeseries() throws Exception "bitwiseXor(\"__acc\", \"a2\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -17873,7 +17873,7 @@ public void testBitwiseAggregatorsGroupBy() throws Exception "bitwiseAnd(\"__acc\", \"a0\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -17890,7 +17890,7 @@ public void testBitwiseAggregatorsGroupBy() throws Exception "bitwiseOr(\"__acc\", \"a1\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -17907,7 +17907,7 @@ public void testBitwiseAggregatorsGroupBy() throws Exception "bitwiseXor(\"__acc\", \"a2\")", null, null, - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -17969,7 +17969,7 @@ public void testStringAgg() throws Exception "array_concat(\"__acc\", \"a0\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("dim1", null, null)) @@ -17986,7 +17986,7 @@ public void testStringAgg() throws Exception "array_set_add_all(\"__acc\", \"a1\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("dim1", null, null)) @@ -18003,7 +18003,7 @@ public void testStringAgg() throws Exception "array_set_add_all(\"__acc\", \"a2\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), and( @@ -18049,7 +18049,7 @@ public void testStringAggMultiValue() throws Exception "array_concat(\"__acc\", \"a0\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("dim3", null, null)) @@ -18066,7 +18066,7 @@ public void testStringAggMultiValue() throws Exception "array_set_add_all(\"__acc\", \"a1\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("dim3", null, null)) @@ -18109,7 +18109,7 @@ public void testStringAggNumeric() throws Exception "array_concat(\"__acc\", \"a0\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -18126,7 +18126,7 @@ public void testStringAggNumeric() throws Exception "array_set_add_all(\"__acc\", \"a1\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("l1", null, null)) @@ -18143,7 +18143,7 @@ public void testStringAggNumeric() throws Exception "array_concat(\"__acc\", \"a2\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("d1", null, null)) @@ -18160,7 +18160,7 @@ public void testStringAggNumeric() throws Exception "array_set_add_all(\"__acc\", \"a3\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("d1", null, null)) @@ -18177,7 +18177,7 @@ public void testStringAggNumeric() throws Exception "array_concat(\"__acc\", \"a4\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("f1", null, null)) @@ -18194,7 +18194,7 @@ public void testStringAggNumeric() throws Exception "array_set_add_all(\"__acc\", \"a5\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("f1", null, null)) @@ -18231,7 +18231,7 @@ public void testStringAggExpression() throws Exception { cannotVectorize(); testQuery( - "SELECT STRING_AGG(DISTINCT CONCAT(dim1, dim2), ',') FROM foo", + "SELECT STRING_AGG(DISTINCT CONCAT(dim1, dim2), ','), STRING_AGG(DISTINCT CONCAT(dim1, dim2), CONCAT('|', '|')) FROM foo", ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) @@ -18254,7 +18254,24 @@ public void testStringAggExpression() throws Exception "array_set_add_all(\"__acc\", \"a0\")", null, "if(array_length(o) == 0, null, array_to_string(o, ','))", - new HumanReadableBytes(1024), + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, + TestExprMacroTable.INSTANCE + ), + not(selector("v0", null, null)) + ), + new FilteredAggregatorFactory( + new ExpressionLambdaAggregatorFactory( + "a1", + ImmutableSet.of("v0"), + "__acc", + "[]", + "[]", + true, + "array_set_add(\"__acc\", \"v0\")", + "array_set_add_all(\"__acc\", \"a1\")", + null, + "if(array_length(o) == 0, null, array_to_string(o, '||'))", + ExpressionLambdaAggregatorFactory.DEFAULT_MAX_SIZE_BYTES, TestExprMacroTable.INSTANCE ), not(selector("v0", null, null)) @@ -18265,11 +18282,21 @@ public void testStringAggExpression() throws Exception .build() ), ImmutableList.of( - useDefault ? new Object[]{"1a,a,2,abc,10.1,defabc"} : new Object[]{"1a,a,2,defabc"} + useDefault ? new Object[]{"1a,a,2,abc,10.1,defabc", "1a||a||2||abc||10.1||defabc"} : new Object[]{"1a,a,2,defabc", "1a||a||2||defabc"} ) ); } + @Test(expected = RelOptPlanner.CannotPlanException.class) + public void testStringAggExpressionNonConstantSeparator() throws Exception + { + testQuery( + "SELECT STRING_AGG(DISTINCT CONCAT(dim1, dim2), CONCAT('|', dim1)) FROM foo", + ImmutableList.of(), + ImmutableList.of() + ); + } + @Test public void testStringAggMaxBytes() throws Exception {