From 262aa0e5ff9533c52b6bb2688efbbc409c563cea Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 6 Apr 2020 23:01:40 -0700 Subject: [PATCH 01/22] better type tracking: add typed postaggs, finalized types for agg factories --- .../data/input/impl/DimensionSchema.java | 32 +-- .../input/impl/DoubleDimensionSchema.java | 1 + .../data/input/impl/FloatDimensionSchema.java | 1 + .../data/input/impl/LongDimensionSchema.java | 2 + .../input/impl/NewSpatialDimensionSchema.java | 1 + .../input/impl/StringDimensionSchema.java | 1 + .../druid/segment/column/ValueType.java | 68 +++++++ .../MomentSketchMaxPostAggregator.java | 7 + .../MomentSketchMinPostAggregator.java | 7 + .../MomentSketchQuantilePostAggregator.java | 7 + ...omentSketchAggregatorFactorySerdeTest.java | 44 ---- .../MomentSketchAggregatorFactoryTest.java | 89 ++++++++ ...TDigestSketchToQuantilePostAggregator.java | 7 + ...DigestSketchToQuantilesPostAggregator.java | 7 + .../TDigestSketchAggregatorFactoryTest.java | 65 ++++++ .../TimestampAggregatorFactory.java | 9 +- .../TimestampMinMaxAggregatorFactoryTest.java | 70 +++++++ .../hll/HllSketchAggregatorFactory.java | 7 + .../HllSketchToEstimatePostAggregator.java | 7 + ...tchToEstimateWithBoundsPostAggregator.java | 7 + .../hll/HllSketchToStringPostAggregator.java | 7 + .../hll/HllSketchUnionPostAggregator.java | 6 + .../DoublesSketchAggregatorFactory.java | 6 + .../DoublesSketchToCDFPostAggregator.java | 7 + ...oublesSketchToHistogramPostAggregator.java | 6 + ...DoublesSketchToQuantilePostAggregator.java | 7 + ...oublesSketchToQuantilesPostAggregator.java | 7 + .../DoublesSketchToRankPostAggregator.java | 7 + .../DoublesSketchToStringPostAggregator.java | 7 + .../theta/SketchConstantPostAggregator.java | 6 + .../theta/SketchEstimatePostAggregator.java | 7 + .../theta/SketchMergeAggregatorFactory.java | 13 ++ .../theta/SketchSetPostAggregator.java | 6 + .../theta/SketchToStringPostAggregator.java | 7 + ...ArrayOfDoublesSketchAggregatorFactory.java | 7 + ...rayOfDoublesSketchSetOpPostAggregator.java | 6 + ...rayOfDoublesSketchTTestPostAggregator.java | 7 + ...etchToEstimateAndBoundsPostAggregator.java | 7 + ...DoublesSketchToEstimatePostAggregator.java | 7 + ...yOfDoublesSketchToMeansPostAggregator.java | 7 + ...ublesSketchToNumEntriesPostAggregator.java | 7 + ...SketchToQuantilesSketchPostAggregator.java | 6 + ...OfDoublesSketchToStringPostAggregator.java | 7 + ...oublesSketchToVariancesPostAggregator.java | 7 + .../hll/HllSketchAggregatorFactoryTest.java | 81 ++++++++ .../DoublesSketchAggregatorFactoryTest.java | 71 +++++++ .../DoublesSketchToCDFPostAggregatorTest.java | 36 ++++ ...esSketchToHistogramPostAggregatorTest.java | 36 ++++ ...esSketchToQuantilesPostAggregatorTest.java | 36 ++++ ...DoublesSketchToRankPostAggregatorTest.java | 36 ++++ .../theta/SketchAggregatorFactoryTest.java | 91 +++++++++ ...yOfDoublesSketchAggregatorFactoryTest.java | 45 ++++ ...fDoublesSketchSetOpPostAggregatorTest.java | 42 ++++ ...fDoublesSketchTTestPostAggregatorTest.java | 39 ++++ ...ToEstimateAndBoundsPostAggregatorTest.java | 1 - ...lesSketchToEstimatePostAggregatorTest.java | 35 ++++ ...oublesSketchToMeansPostAggregatorTest.java | 35 ++++ ...sSketchToNumEntriesPostAggregatorTest.java | 35 ++++ ...chToQuantilesSketchPostAggregatorTest.java | 37 ++++ ...ublesSketchToStringPostAggregatorTest.java | 35 ++++ ...esSketchToVariancesPostAggregatorTest.java | 35 ++++ .../BloomFilterAggregatorFactoryTest.java | 72 +++++++ ...ApproximateHistogramAggregatorFactory.java | 7 + .../histogram/BucketsPostAggregator.java | 6 + .../CustomBucketsPostAggregator.java | 6 + .../histogram/EqualBucketsPostAggregator.java | 6 + ...ixedBucketsHistogramAggregatorFactory.java | 7 + .../histogram/MaxPostAggregator.java | 7 + .../histogram/MinPostAggregator.java | 7 + .../histogram/QuantilePostAggregator.java | 7 + .../histogram/QuantilesPostAggregator.java | 6 + ...pproximateHistogramPostAggregatorTest.java | 42 ++++ .../PvaluefromZscorePostAggregator.java | 7 + .../teststats/ZtestPostAggregator.java | 7 + .../StandardDeviationPostAggregator.java | 7 + .../variance/VarianceAggregatorFactory.java | 6 + .../VarianceAggregatorFactoryTest.java | 71 +++++++ .../apache/druid/indexer/InputRowSerde.java | 40 ++-- .../druid/indexer/InputRowSerdeTest.java | 3 + .../common/task/CompactionTaskTest.java | 3 +- .../org/apache/druid/query/DruidMetrics.java | 5 +- .../query/aggregation/AggregatorFactory.java | 34 +++- .../query/aggregation/BufferAggregator.java | 6 +- .../aggregation/CountAggregatorFactory.java | 3 +- .../FilteredAggregatorFactory.java | 6 + .../HistogramAggregatorFactory.java | 6 + .../JavaScriptAggregatorFactory.java | 3 +- .../query/aggregation/PostAggregator.java | 9 + .../SimpleDoubleAggregatorFactory.java | 4 +- .../SimpleFloatAggregatorFactory.java | 2 +- .../SimpleLongAggregatorFactory.java | 2 +- .../SuppressedAggregatorFactory.java | 6 + .../any/DoubleAnyAggregatorFactory.java | 3 +- .../any/FloatAnyAggregatorFactory.java | 3 +- .../any/LongAnyAggregatorFactory.java | 3 +- .../any/StringAnyAggregatorFactory.java | 3 +- .../CardinalityAggregatorFactory.java | 7 + .../first/DoubleFirstAggregatorFactory.java | 11 +- .../first/FloatFirstAggregatorFactory.java | 9 +- .../first/LongFirstAggregatorFactory.java | 9 +- .../first/StringFirstAggregatorFactory.java | 7 + .../HyperUniqueFinalizingPostAggregator.java | 23 ++- .../HyperUniquesAggregatorFactory.java | 6 + .../last/DoubleLastAggregatorFactory.java | 11 +- .../last/FloatLastAggregatorFactory.java | 9 +- .../last/LongLastAggregatorFactory.java | 9 +- .../last/StringLastAggregatorFactory.java | 7 + .../mean/DoubleMeanAggregatorFactory.java | 7 + .../post/ArithmeticPostAggregator.java | 7 + .../post/ConstantPostAggregator.java | 7 + .../post/DoubleGreatestPostAggregator.java | 7 + .../post/DoubleLeastPostAggregator.java | 7 + .../post/ExpressionPostAggregator.java | 41 ++-- .../post/FieldAccessPostAggregator.java | 29 ++- .../FinalizingFieldAccessPostAggregator.java | 22 +- .../post/JavaScriptPostAggregator.java | 7 + .../post/LongGreatestPostAggregator.java | 7 + .../post/LongLeastPostAggregator.java | 7 + .../GrouperBufferComparatorUtils.java | 5 +- .../epinephelinae/RowBasedGrouperHelper.java | 3 +- .../QueryableIndexColumnSelectorFactory.java | 3 +- .../druid/segment/column/RowSignature.java | 25 +-- .../segment/column/SimpleColumnHolder.java | 2 +- .../druid/segment/column/ValueType.java | 140 ------------- .../druid/segment/column/ValueTypes.java | 94 +++++++++ .../segment/incremental/IncrementalIndex.java | 42 +--- ...IncrementalIndexColumnSelectorFactory.java | 4 +- .../aggregation/AggregatorFactoryTest.java | 192 +++++++++++++++++- ...perUniqueFinalizingPostAggregatorTest.java | 39 ++++ .../post/ArithmeticPostAggregatorTest.java | 43 ++++ .../post/ConstantPostAggregatorTest.java | 37 ++++ .../DoubleGreatestPostAggregatorTest.java | 41 ++++ .../post/DoubleLeastPostAggregatorTest.java | 41 ++++ .../post/FieldAccessPostAggregatorTest.java | 43 ++++ ...nalizingFieldAccessPostAggregatorTest.java | 43 +++- .../post/JavaScriptPostAggregatorTest.java | 48 ++++- .../post/LongGreatestPostAggregatorTest.java | 41 ++++ .../post/LongLeastPostAggregatorTest.java | 41 ++++ ...ByLimitPushDownInsufficientBufferTest.java | 3 +- .../GroupByQueryQueryToolChestTest.java | 4 +- .../TimeseriesQueryQueryToolChestTest.java | 2 +- .../topn/TopNQueryQueryToolChestTest.java | 2 +- 142 files changed, 2568 insertions(+), 373 deletions(-) create mode 100644 core/src/main/java/org/apache/druid/segment/column/ValueType.java delete mode 100644 extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactorySerdeTest.java create mode 100644 extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java create mode 100644 extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java create mode 100644 extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampMinMaxAggregatorFactoryTest.java create mode 100644 extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactoryTest.java create mode 100644 extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java create mode 100644 extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java create mode 100644 extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactoryTest.java delete mode 100644 processing/src/main/java/org/apache/druid/segment/column/ValueType.java create mode 100644 processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java diff --git a/core/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/core/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index 66461777ce32..263f619a1dbe 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -29,6 +29,7 @@ import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.segment.column.ValueType; import java.util.Objects; @@ -52,37 +53,6 @@ public abstract class DimensionSchema public static final String DOUBLE_TYPE_NAME = "double"; private static final EmittingLogger log = new EmittingLogger(DimensionSchema.class); - - // main druid and druid-api should really use the same ValueType enum. - // merge them when druid-api is merged back into the main repo - - /** - * Should be the same as {@code org.apache.druid.segment.column.ValueType}. - * TODO merge them when druid-api is merged back into the main repo - */ - public enum ValueType - { - FLOAT, - LONG, - STRING, - DOUBLE, - @SuppressWarnings("unused") // used in org.apache.druid.segment.column.ValueType - COMPLEX; - - @JsonValue - @Override - public String toString() - { - return StringUtils.toUpperCase(this.name()); - } - - @JsonCreator - public static ValueType fromString(String name) - { - return valueOf(StringUtils.toUpperCase(name)); - } - } - public enum MultiValueHandling { SORTED_ARRAY, diff --git a/core/src/main/java/org/apache/druid/data/input/impl/DoubleDimensionSchema.java b/core/src/main/java/org/apache/druid/data/input/impl/DoubleDimensionSchema.java index e00e86ab100c..7e45d1e89aab 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/DoubleDimensionSchema.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/DoubleDimensionSchema.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.ValueType; public class DoubleDimensionSchema extends DimensionSchema { diff --git a/core/src/main/java/org/apache/druid/data/input/impl/FloatDimensionSchema.java b/core/src/main/java/org/apache/druid/data/input/impl/FloatDimensionSchema.java index 86bb1cd24bce..a9a3ad66770a 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/FloatDimensionSchema.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/FloatDimensionSchema.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.ValueType; public class FloatDimensionSchema extends DimensionSchema { diff --git a/core/src/main/java/org/apache/druid/data/input/impl/LongDimensionSchema.java b/core/src/main/java/org/apache/druid/data/input/impl/LongDimensionSchema.java index a342013dc36e..6af9d4ebfb50 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/LongDimensionSchema.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/LongDimensionSchema.java @@ -22,6 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.ValueType; + public class LongDimensionSchema extends DimensionSchema { diff --git a/core/src/main/java/org/apache/druid/data/input/impl/NewSpatialDimensionSchema.java b/core/src/main/java/org/apache/druid/data/input/impl/NewSpatialDimensionSchema.java index 00bb6b82f1af..021899590e17 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/NewSpatialDimensionSchema.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/NewSpatialDimensionSchema.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.ValueType; import java.util.List; diff --git a/core/src/main/java/org/apache/druid/data/input/impl/StringDimensionSchema.java b/core/src/main/java/org/apache/druid/data/input/impl/StringDimensionSchema.java index 7c030eb703a3..7a83e0f8b06d 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/StringDimensionSchema.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/StringDimensionSchema.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.ValueType; public class StringDimensionSchema extends DimensionSchema { diff --git a/core/src/main/java/org/apache/druid/segment/column/ValueType.java b/core/src/main/java/org/apache/druid/segment/column/ValueType.java new file mode 100644 index 000000000000..219f7dc63d42 --- /dev/null +++ b/core/src/main/java/org/apache/druid/segment/column/ValueType.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.column; + +import com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.druid.java.util.common.StringUtils; + +import javax.annotation.Nullable; + +public enum ValueType +{ + DOUBLE, + FLOAT, + LONG, + STRING, + COMPLEX, + DOUBLE_ARRAY, + LONG_ARRAY, + STRING_ARRAY; + + + public boolean isNumeric() + { + return isNumeric(this); + } + + public boolean isPrimitiveScalar() + { + return this.equals(ValueType.STRING) || isNumeric(this); + } + + public boolean isComplex() + { + return !isPrimitiveScalar(); + } + + @Nullable + @JsonCreator + public static ValueType fromString(@Nullable String name) + { + if (name == null) { + return null; + } + return valueOf(StringUtils.toUpperCase(name)); + } + + public static boolean isNumeric(ValueType type) + { + return type == ValueType.LONG || type == ValueType.FLOAT || type == ValueType.DOUBLE; + } +} diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java index 38755a6b7022..d91e650564f2 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -57,6 +58,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java index b244243b5057..96875203ad34 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -56,6 +57,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java index 81f5b63acf08..28a6af1f262c 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.momentsketch.MomentSketchWrapper; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -61,6 +62,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactorySerdeTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactorySerdeTest.java deleted file mode 100644 index 7c00ea80be2a..000000000000 --- a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactorySerdeTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation.momentsketch.aggregator; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.junit.Assert; -import org.junit.Test; - -public class MomentSketchAggregatorFactorySerdeTest -{ - @Test - public void serializeDeserializeFactoryWithFieldName() throws Exception - { - ObjectMapper objectMapper = new DefaultObjectMapper(); - MomentSketchAggregatorFactory factory = new MomentSketchAggregatorFactory( - "name", "fieldName", 128, true - ); - - MomentSketchAggregatorFactory other = objectMapper.readValue( - objectMapper.writeValueAsString(factory), - MomentSketchAggregatorFactory.class - ); - - Assert.assertEquals(factory, other); - } -} diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java new file mode 100644 index 000000000000..5a1991a7a62b --- /dev/null +++ b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.momentsketch.aggregator; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class MomentSketchAggregatorFactoryTest +{ + @Test + public void serializeDeserializeFactoryWithFieldName() throws Exception + { + ObjectMapper objectMapper = new DefaultObjectMapper(); + MomentSketchAggregatorFactory factory = new MomentSketchAggregatorFactory( + "name", "fieldName", 128, true + ); + + MomentSketchAggregatorFactory other = objectMapper.readValue( + objectMapper.writeValueAsString(factory), + MomentSketchAggregatorFactory.class + ); + + Assert.assertEquals(factory, other); + } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new MomentSketchAggregatorFactory("moment", "col", null, null), + new MomentSketchMergeAggregatorFactory("momentMerge", null, null) + ) + .postAggregators( + new FieldAccessPostAggregator("moment-access", "moment"), + new FinalizingFieldAccessPostAggregator("moment-finalize", "moment"), + new FieldAccessPostAggregator("momentMerge-access", "momentMerge"), + new FinalizingFieldAccessPostAggregator("momentMerge-finalize", "momentMerge") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("moment", null) + .add("momentMerge", null) + .add("moment-access", ValueType.COMPLEX) + .add("moment-finalize", ValueType.COMPLEX) + .add("momentMerge-access", ValueType.COMPLEX) + .add("momentMerge-finalize", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java index d54b90b4ac89..4a41efd8dbfd 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -67,6 +68,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java index 245f648cc3cf..ebd9021eafcc 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -67,6 +68,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java new file mode 100644 index 000000000000..82e482530e75 --- /dev/null +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.tdigestsketch; + +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class TDigestSketchAggregatorFactoryTest +{ + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new TDigestSketchAggregatorFactory("tdigest", "col", null) + ) + .postAggregators( + new FieldAccessPostAggregator("tdigest-access", "tdigest"), + new FinalizingFieldAccessPostAggregator("tdigest-finalize", "tdigest") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("tdigest", null) + .add("tdigest-access", ValueType.COMPLEX) + .add("tdigest-finalize", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java b/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java index b161f3449e34..d94d265508d1 100644 --- a/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java +++ b/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java @@ -25,6 +25,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -202,7 +203,13 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return "long"; + return ValueType.LONG.toString(); + } + + @Override + public String getFinalizedTypeName() + { + return "dateTime"; } @Override diff --git a/extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampMinMaxAggregatorFactoryTest.java b/extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampMinMaxAggregatorFactoryTest.java new file mode 100644 index 000000000000..c8424c3319fc --- /dev/null +++ b/extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampMinMaxAggregatorFactoryTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class TimestampMinMaxAggregatorFactoryTest +{ + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new TimestampMaxAggregatorFactory("timeMax", "__time", null), + new TimestampMinAggregatorFactory("timeMin", "__time", null) + ) + .postAggregators( + new FieldAccessPostAggregator("timeMax-access", "timeMax"), + new FinalizingFieldAccessPostAggregator("timeMax-finalize", "timeMax"), + new FieldAccessPostAggregator("timeMin-access", "timeMin"), + new FinalizingFieldAccessPostAggregator("timeMin-finalize", "timeMin") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("timeMax", null) + .add("timeMin", null) + .add("timeMax-access", ValueType.LONG) + .add("timeMax-finalize", ValueType.COMPLEX) + .add("timeMin-access", ValueType.LONG) + .add("timeMin-finalize", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java index 06a4f8f160dc..6485c22354de 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.ObjectAggregateCombiner; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Collections; @@ -168,6 +169,12 @@ public Class classOfObject() }; } + @Override + public String getFinalizedTypeName() + { + return round ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + } + @Nullable @Override public Object finalizeComputation(@Nullable final Object object) diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java index 646779d0c529..9f09917dc3e2 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -62,6 +63,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java index 81983181ae11..e87f2b1ddfd9 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Comparator; @@ -68,6 +69,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java index d5015bcccf46..a4d925d1a836 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java @@ -26,6 +26,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -79,6 +80,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.STRING.toString(); + } + @Override public PostAggregator decorate(final Map aggregators) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java index 4a5518bee788..aded8f7ba943 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java @@ -70,6 +70,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return "hllSketch"; + } + @JsonProperty public List getFields() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 3c154d040ceb..e02ac9704023 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -253,6 +253,12 @@ public String getTypeName() return DoublesSketchModule.DOUBLES_SKETCH; } + @Override + public String getFinalizedTypeName() + { + return ValueType.LONG.toString(); + } + @Override public byte[] getCacheKey() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java index 58edb73495ec..ac29a6f28a7a 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -71,6 +72,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java index eb135a2656d1..f069a3d2c97b 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java @@ -75,6 +75,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return "doublesSketch"; + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java index e15a5000061d..3af11295a2c9 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -58,6 +59,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java index 4961f022a8b1..7f73b9455119 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -59,6 +60,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java index fae14b777257..f795a24fe60c 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -58,6 +59,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java index 553d1d23afee..659d24034cb7 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -55,6 +56,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.STRING.toString(); + } + @JsonProperty public PostAggregator getField() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java index 3186fcec03a2..a9aa253da423 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java @@ -77,6 +77,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return SketchModule.THETA_SKETCH; + } + @Override public SketchConstantPostAggregator decorate(Map aggregators) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java index a725c64b48ba..750708ae1067 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -100,6 +101,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java index 4d9af8c7cbdf..3679e191dfb2 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java @@ -24,6 +24,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactoryNotMergeableException; import org.apache.druid.query.aggregation.AggregatorUtil; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Collections; @@ -147,6 +148,18 @@ public String getTypeName() } } + @Override + public String getFinalizedTypeName() + { + if (shouldFinalize) { + if (errorBoundsStdDev != null) { + return "estimateWithErrorBounds"; + } + return ValueType.DOUBLE.toString(); + } + return getTypeName(); + } + @Override public boolean equals(Object o) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java index b0e311e9c966..f4c2d0a7d108 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java @@ -94,6 +94,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return SketchModule.THETA_SKETCH; + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java index b34b239738ff..77f523737510 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java @@ -25,6 +25,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -78,6 +79,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.STRING.toString(); + } + @Override public PostAggregator decorate(final Map aggregators) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java index 1e851b32a920..8688a9363981 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java @@ -41,6 +41,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DimensionSelector; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.ArrayList; @@ -301,6 +302,12 @@ public String getTypeName() return ArrayOfDoublesSketchModule.ARRAY_OF_DOUBLES_SKETCH_BUILD_AGG; } + @Override + public String getFinalizedTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public boolean equals(final Object o) { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java index 06b970d6944c..8aaf3c171037 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java @@ -81,6 +81,12 @@ public ArrayOfDoublesSketch compute(final Map combinedAggregator return operation.apply(nominalEntries, numberOfValues, sketches); } + @Override + public String getTypeName() + { + return "arrayOfDoublesSketch"; + } + @JsonProperty public String getOperation() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java index a0ad76cd07b5..b696c7b10457 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -86,6 +87,12 @@ public double[] compute(final Map combinedAggregators) return pValues; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + private static SummaryStatistics[] getStats(final ArrayOfDoublesSketch sketch) { final SummaryStatistics[] stats = new SummaryStatistics[sketch.getNumValues()]; diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java index 8ded7a3955c9..5ef115201444 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java @@ -26,6 +26,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Comparator; @@ -68,6 +69,12 @@ public double[] compute(final Map combinedAggregators) return new double[] {sketch.getEstimate(), sketch.getLowerBound(numStdDevs), sketch.getUpperBound(numStdDevs)}; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java index a3cef1b56557..8ed74bf12ef0 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java @@ -25,6 +25,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -51,6 +52,12 @@ public Double compute(final Map combinedAggregators) return sketch.getEstimate(); } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java index 2f99eda2e0cb..9cbab054e05a 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -67,6 +68,12 @@ public double[] compute(final Map combinedAggregators) return means; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java index 23f325394fee..f2b862cf3ce6 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java @@ -25,6 +25,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -51,6 +52,12 @@ public Integer compute(final Map combinedAggregators) return sketch.getRetainedEntries(); } + @Override + public String getTypeName() + { + return ValueType.LONG.toString(); + } + @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java index f294781d0e8c..e48c6932f07b 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java @@ -82,6 +82,12 @@ public DoublesSketch compute(final Map combinedAggregators) return qs; } + @Override + public String getTypeName() + { + return "doublesSketch"; + } + @JsonProperty public int getColumn() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java index 316b6dc8a025..eb0b1a48afbd 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java @@ -26,6 +26,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -54,6 +55,12 @@ public String compute(final Map combinedAggregators) return sketch.toString(); } + @Override + public String getTypeName() + { + return ValueType.STRING.toString(); + } + @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java index a5a1e5a1cb2b..10ece6af754a 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -67,6 +68,12 @@ public double[] compute(final Map combinedAggregators) return variances; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE_ARRAY.toString(); + } + @Override public Comparator getComparator() { diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java index f4fb74a3c977..5ad3f2967177 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java @@ -21,10 +21,19 @@ import org.apache.datasketches.hll.HllSketch; import org.apache.datasketches.hll.TgtHllType; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Before; @@ -220,6 +229,78 @@ public void testToString() } } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new HllSketchBuildAggregatorFactory( + "hllBuild", + "col", + null, + null, + false + ), + new HllSketchBuildAggregatorFactory( + "hllBuildRound", + "col", + null, + null, + true + ), + new HllSketchMergeAggregatorFactory( + "hllMerge", + "col", + null, + null, + false + ), + new HllSketchMergeAggregatorFactory( + "hllMergeRound", + "col", + null, + null, + true + ) + ) + .postAggregators( + new FieldAccessPostAggregator("hllBuild-access", "hllBuild"), + new FinalizingFieldAccessPostAggregator("hllBuild-finalize", "hllBuild"), + new FieldAccessPostAggregator("hllBuildRound-access", "hllBuildRound"), + new FinalizingFieldAccessPostAggregator("hllBuildRound-finalize", "hllBuildRound"), + new FieldAccessPostAggregator("hllMerge-access", "hllMerge"), + new FinalizingFieldAccessPostAggregator("hllMerge-finalize", "hllMerge"), + new FieldAccessPostAggregator("hllMergeRound-access", "hllMergeRound"), + new FinalizingFieldAccessPostAggregator("hllMergeRound-finalize", "hllMergeRound") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("hllBuild", null) + .add("hllBuildRound", null) + .add("hllMerge", null) + .add("hllMergeRound", null) + .add("hllBuild-access", ValueType.COMPLEX) + .add("hllBuild-finalize", ValueType.DOUBLE) + .add("hllBuildRound-access", ValueType.COMPLEX) + .add("hllBuildRound-finalize", ValueType.LONG) + .add("hllMerge-access", ValueType.COMPLEX) + .add("hllMerge-finalize", ValueType.DOUBLE) + .add("hllMergeRound-access", ValueType.COMPLEX) + .add("hllMergeRound-finalize", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } + private static boolean isToStringField(Field field) { int modfiers = field.getModifiers(); diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactoryTest.java new file mode 100644 index 000000000000..0b5f80ee9380 --- /dev/null +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactoryTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.datasketches.quantiles; + +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class DoublesSketchAggregatorFactoryTest +{ + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new DoublesSketchAggregatorFactory("doublesSketch", "col", 8), + new DoublesSketchMergeAggregatorFactory("doublesSketchMerge", 8) + ) + .postAggregators( + new FieldAccessPostAggregator("doublesSketch-access", "doublesSketch"), + new FinalizingFieldAccessPostAggregator("doublesSketch-finalize", "doublesSketch"), + new FieldAccessPostAggregator("doublesSketchMerge-access", "doublesSketchMerge"), + new FinalizingFieldAccessPostAggregator("doublesSketchMerge-finalize", "doublesSketchMerge") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("doublesSketch", null) + .add("doublesSketchMerge", null) + .add("doublesSketch-access", ValueType.COMPLEX) + .add("doublesSketch-finalize", ValueType.LONG) + .add("doublesSketchMerge-access", ValueType.COMPLEX) + .add("doublesSketchMerge-finalize", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregatorTest.java index f41f3167083a..e22093502bea 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregatorTest.java @@ -19,10 +19,16 @@ package org.apache.druid.query.aggregation.datasketches.quantiles; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.TestDoubleColumnSelectorImpl; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -81,4 +87,34 @@ public void normalCase() Assert.assertEquals(0.5, cdf[0], 0); Assert.assertEquals(1.0, cdf[1], 0); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new DoublesSketchAggregatorFactory("sketch", "col", 8) + ) + .postAggregators( + new DoublesSketchToCDFPostAggregator( + "a", + new FieldAccessPostAggregator("field", "sketch"), + new double[] {4} + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("sketch", null) + .add("a", ValueType.DOUBLE_ARRAY) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java index b5aeec985fb3..0d96293f231b 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java @@ -19,10 +19,16 @@ package org.apache.druid.query.aggregation.datasketches.quantiles; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.TestDoubleColumnSelectorImpl; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -81,4 +87,34 @@ public void normalCase() Assert.assertEquals(3.0, histogram[0], 0); Assert.assertEquals(3.0, histogram[1], 0); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new DoublesSketchAggregatorFactory("sketch", "col", 8) + ) + .postAggregators( + new DoublesSketchToHistogramPostAggregator( + "a", + new FieldAccessPostAggregator("field", "sketch"), + new double[] {3.5} + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("sketch", null) + .add("a", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregatorTest.java index 7a4edab5e443..525c4a5d2fc8 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregatorTest.java @@ -19,10 +19,16 @@ package org.apache.druid.query.aggregation.datasketches.quantiles; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.TestDoubleColumnSelectorImpl; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -83,4 +89,34 @@ public void normalCase() Assert.assertEquals(3.0, quantiles[1], 0); Assert.assertEquals(5.0, quantiles[2], 0); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new DoublesSketchAggregatorFactory("sketch", "col", 8) + ) + .postAggregators( + new DoublesSketchToQuantilesPostAggregator( + "a", + new FieldAccessPostAggregator("field", "sketch"), + new double[] {0, 0.5, 1} + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("sketch", null) + .add("a", ValueType.DOUBLE_ARRAY) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregatorTest.java index f9ffccc1602c..f235cca6c998 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregatorTest.java @@ -19,10 +19,16 @@ package org.apache.druid.query.aggregation.datasketches.quantiles; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.TestDoubleColumnSelectorImpl; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -75,4 +81,34 @@ public void normalCase() final double rank = (double) postAgg.compute(fields); Assert.assertEquals(0.5, rank, 0); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new DoublesSketchAggregatorFactory("sketch", "col", 8) + ) + .postAggregators( + new DoublesSketchToRankPostAggregator( + "a", + new FieldAccessPostAggregator("field", "sketch"), + 4 + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("sketch", null) + .add("a", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java new file mode 100644 index 000000000000..9299fbe8f6e3 --- /dev/null +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.datasketches.theta; + +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.theta.oldapi.OldSketchBuildAggregatorFactory; +import org.apache.druid.query.aggregation.datasketches.theta.oldapi.OldSketchMergeAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class SketchAggregatorFactoryTest +{ + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new OldSketchBuildAggregatorFactory("oldBuild", "col", 16), + new OldSketchMergeAggregatorFactory("oldMerge", "col", 16, false), + new OldSketchMergeAggregatorFactory("oldMergeFinalize", "col", 16, true), + new SketchMergeAggregatorFactory("merge", "col", 16, false, false, null), + new SketchMergeAggregatorFactory("mergeFinalize", "col", 16, true, false, null) + ) + .postAggregators( + new FieldAccessPostAggregator("oldBuild-access", "oldBuild"), + new FinalizingFieldAccessPostAggregator("oldBuild-finalize", "oldBuild"), + new FieldAccessPostAggregator("oldMerge-access", "oldMerge"), + new FinalizingFieldAccessPostAggregator("oldMerge-finalize", "oldMerge"), + new FieldAccessPostAggregator("oldMergeFinalize-access", "oldMergeFinalize"), + new FinalizingFieldAccessPostAggregator("oldMergeFinalize-finalize", "oldMergeFinalize"), + new FieldAccessPostAggregator("merge-access", "merge"), + new FinalizingFieldAccessPostAggregator("merge-finalize", "merge"), + new FieldAccessPostAggregator("mergeFinalize-access", "mergeFinalize"), + new FinalizingFieldAccessPostAggregator("mergeFinalize-finalize", "mergeFinalize") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("oldBuild", null) + .add("oldMerge", null) + .add("oldMergeFinalize", null) + .add("merge", null) + .add("mergeFinalize", null) + .add("oldBuild-access", ValueType.COMPLEX) + .add("oldBuild-finalize", ValueType.DOUBLE) + .add("oldMerge-access", ValueType.COMPLEX) + .add("oldMerge-finalize", ValueType.COMPLEX) + .add("oldMergeFinalize-access", ValueType.COMPLEX) + .add("oldMergeFinalize-finalize", ValueType.DOUBLE) + .add("merge-access", ValueType.COMPLEX) + .add("merge-finalize", ValueType.COMPLEX) + .add("mergeFinalize-access", ValueType.COMPLEX) + .add("mergeFinalize-finalize", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java index 1bc408bf9be1..f1115425d324 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java @@ -23,9 +23,18 @@ import org.apache.datasketches.tuple.ArrayOfDoublesSketch; import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch; import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.TestObjectColumnSelector; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -76,4 +85,40 @@ public void testEquals() Assert.assertEquals(a1, a2); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new ArrayOfDoublesSketchAggregatorFactory( + "arrayOfDoublesSketch", + "col", + 1, + ImmutableList.of("met"), + 1 + ) + ) + .postAggregators( + new FieldAccessPostAggregator("a", "arrayOfDoublesSketch"), + new FinalizingFieldAccessPostAggregator("b", "arrayOfDoublesSketch") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("arrayOfDoublesSketch", null) + .add("a", ValueType.COMPLEX) + .add("b", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregatorTest.java index 4b416eb389d3..e81c46e0123e 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregatorTest.java @@ -19,8 +19,16 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -86,4 +94,38 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchSetOpPostAggregator( + "a", + "UNION", + null, + null, + ImmutableList.of( + new ConstantPostAggregator("", 0), + new ConstantPostAggregator("", 0) + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregatorTest.java index 6ca780ed9f08..2b5a382e3d99 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregatorTest.java @@ -19,8 +19,16 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -77,4 +85,35 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchTTestPostAggregator( + "a", + ImmutableList.of( + new ConstantPostAggregator("", 0), + new ConstantPostAggregator("", 0) + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.DOUBLE_ARRAY) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregatorTest.java index 5b9220dee6cd..01d359d6b977 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregatorTest.java @@ -75,5 +75,4 @@ public void equalsAndHashCode() ); Assert.assertFalse(postAgg1.equals(postAgg5)); } - } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregatorTest.java index b87fda4b13ae..e806e2629507 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregatorTest.java @@ -19,8 +19,15 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -72,4 +79,32 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchToEstimatePostAggregator( + "a", + new ConstantPostAggregator("", 0) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregatorTest.java index 0ee27c075fe2..94301dbed25f 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregatorTest.java @@ -19,8 +19,15 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -72,4 +79,32 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchToMeansPostAggregator( + "a", + new ConstantPostAggregator("", 0) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.DOUBLE_ARRAY) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregatorTest.java index e22b4655db2f..9f7225ca59c0 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregatorTest.java @@ -19,8 +19,15 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -72,4 +79,32 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchToNumEntriesPostAggregator( + "a", + new ConstantPostAggregator("", 1) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregatorTest.java index 16a4a2206e5b..68407d2330dc 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregatorTest.java @@ -19,8 +19,15 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -80,4 +87,34 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchToQuantilesSketchPostAggregator( + "a", + new ConstantPostAggregator("", 0), + 2, + null + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregatorTest.java index 77c35bdb11c0..496d872299c0 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregatorTest.java @@ -19,8 +19,15 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -72,4 +79,32 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchToStringPostAggregator( + "a", + new ConstantPostAggregator("", 1) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.STRING) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregatorTest.java index f1b06618c829..269e3daa61f5 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregatorTest.java @@ -19,8 +19,15 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -72,4 +79,32 @@ public void equalsAndHashCode() Assert.assertFalse(postAgg1.equals(postAgg5)); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArrayOfDoublesSketchToVariancesPostAggregator( + "a", + new ConstantPostAggregator("", 0) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.DOUBLE_ARRAY) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java new file mode 100644 index 000000000000..f03b12eb2767 --- /dev/null +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.bloom; + +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class BloomFilterAggregatorFactoryTest +{ + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new BloomFilterAggregatorFactory("bloom", DefaultDimensionSpec.of("col"), 1024), + new BloomFilterMergeAggregatorFactory("bloomMerge", "bloom", 1024) + ) + .postAggregators( + new FieldAccessPostAggregator("bloom-access", "bloom"), + new FinalizingFieldAccessPostAggregator("bloom-finalize", "bloom"), + new FieldAccessPostAggregator("bloomMerge-access", "bloomMerge"), + new FinalizingFieldAccessPostAggregator("bloomMerge-finalize", "bloomMerge") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("bloom", null) + .add("bloomMerge", null) + .add("bloom-access", ValueType.COMPLEX) + .add("bloom-finalize", ValueType.COMPLEX) + .add("bloomMerge-access", ValueType.COMPLEX) + .add("bloomMerge-finalize", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index bc1295466764..3567f39b9913 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -35,6 +35,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -304,6 +305,12 @@ public String getTypeName() return "approximateHistogram"; } + @Override + public String getFinalizedTypeName() + { + return finalizeAsBase64Binary ? ValueType.STRING.toString() : "histogram"; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java index c47044cd3307..b4a4dd75cfe7 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java @@ -67,6 +67,12 @@ public Object compute(Map values) return ah.toHistogram(bucketSize, offset); } + @Override + public String getTypeName() + { + return "histogram"; + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java index a9187beba0ee..b71d6c29df1f 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java @@ -63,6 +63,12 @@ public Object compute(Map values) return ah.toHistogram(breaks); } + @Override + public String getTypeName() + { + return "histogram"; + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java index 517a7cc66b62..5e84e3c5e4eb 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java @@ -64,6 +64,12 @@ public Object compute(Map values) return ah.toHistogram(numBuckets); } + @Override + public String getTypeName() + { + return "histogram"; + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java index b8df5b62e0cb..25fd40165a9c 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java @@ -32,6 +32,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Collections; @@ -251,6 +252,12 @@ public String getTypeName() return FixedBucketsHistogramAggregator.TYPE_NAME; } + @Override + public String getFinalizedTypeName() + { + return ValueType.STRING.toString(); + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java index 4c507d58ccdd..8454c9ff7a1e 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -72,6 +73,12 @@ public Object compute(Map values) } } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java index 3c9dd9f500db..baadc4561432 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -74,6 +75,12 @@ public Object compute(Map values) throw new ISE("Unknown value type: " + val.getClass()); } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java index 13640ebbb5fb..adfbd1085d98 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -84,6 +85,12 @@ public Object compute(Map values) } } + @Override + public String getTypeName() + { + return ValueType.FLOAT.toString(); + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java index 2ef4c5059b1a..90314204e017 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java @@ -92,6 +92,12 @@ public Object compute(Map values) throw new ISE("Unknown value type: " + val.getClass()); } + @Override + public String getTypeName() + { + return "quantiles"; + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java index 7783d67e65db..68823dcead81 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java @@ -19,7 +19,13 @@ package org.apache.druid.query.aggregation.histogram; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.TestFloatColumnSelector; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; @@ -63,4 +69,40 @@ public void testApproxHistogramCompute() ); Assert.assertEquals(ah.toHistogram(5), approximateHistogramPostAggregator.compute(metricValues)); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators(new ApproximateHistogramAggregatorFactory("approxHisto", "col", null, null, null, null, false)) + .postAggregators( + new BucketsPostAggregator("bucket", "approxHisto", 100, 0), + new EqualBucketsPostAggregator("equal", "approxHisto", 5), + new CustomBucketsPostAggregator("custom", "approxHisto", new float[]{1.0f, 20.0f, 75.0f}), + new MinPostAggregator("min", "approxHisto"), + new MaxPostAggregator("max", "approxHisto"), + new QuantilePostAggregator("quantile", "approxHisto", 0.5f), + new QuantilesPostAggregator("quantiles", "approxHisto", new float[]{0.2f, 0.5f, 0.75f}) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("approxHisto", null) + .add("bucket", ValueType.COMPLEX) + .add("equal", ValueType.COMPLEX) + .add("custom", ValueType.COMPLEX) + .add("min", ValueType.DOUBLE) + .add("max", ValueType.DOUBLE) + .add("quantile", ValueType.FLOAT) + .add("quantiles", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java index cf9ff4a9e493..ae4f65414708 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java @@ -31,6 +31,7 @@ import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Collections; import java.util.Comparator; @@ -105,6 +106,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java index 53caa36d04ce..73a22b35e093 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java @@ -30,6 +30,7 @@ import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Collections; import java.util.Comparator; @@ -114,6 +115,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public ZtestPostAggregator decorate(Map aggregators) { diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java index b4f702de06f8..cdd6950d4a7c 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -84,6 +85,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public PostAggregator decorate(Map aggregators) { diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java index 10a04efea6f3..5079134a52d0 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java @@ -92,6 +92,12 @@ public String getTypeName() return VARIANCE_TYPE_NAME; } + @Override + public String getFinalizedTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactoryTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactoryTest.java new file mode 100644 index 000000000000..2e51b2949403 --- /dev/null +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactoryTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.variance; + +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class VarianceAggregatorFactoryTest +{ + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new VarianceAggregatorFactory("variance", "col"), + new VarianceFoldingAggregatorFactory("varianceFold", "col", null) + ) + .postAggregators( + new FieldAccessPostAggregator("variance-access", "variance"), + new FinalizingFieldAccessPostAggregator("variance-finalize", "variance"), + new FieldAccessPostAggregator("varianceFold-access", "varianceFold"), + new FinalizingFieldAccessPostAggregator("varianceFold-finalize", "varianceFold") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("variance", null) + .add("varianceFold", null) + .add("variance-access", ValueType.COMPLEX) + .add("variance-finalize", ValueType.DOUBLE) + .add("varianceFold-access", ValueType.COMPLEX) + .add("varianceFold-finalize", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } +} diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java index c6885b3caa7a..1307c91f6baa 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java @@ -340,22 +340,25 @@ public static SerializeResult toBytes( parseExceptionMessages.add(e.getMessage()); } - String t = aggFactory.getTypeName(); + final ValueType type = aggFactory.getType(); + final String typeName = aggFactory.getTypeName(); + if (agg.isNull()) { out.writeByte(NullHandling.IS_NULL_BYTE); } else { out.writeByte(NullHandling.IS_NOT_NULL_BYTE); - if ("float".equals(t)) { + if (ValueType.FLOAT.equals(type)) { out.writeFloat(agg.getFloat()); - } else if ("long".equals(t)) { + } else if (ValueType.LONG.equals(type)) { WritableUtils.writeVLong(out, agg.getLong()); - } else if ("double".equals(t)) { + } else if (ValueType.DOUBLE.equals(type)) { out.writeDouble(agg.getDouble()); - } else { - //its a complex metric + } else if (ValueType.COMPLEX.equals(type)) { Object val = agg.get(); - ComplexMetricSerde serde = getComplexMetricSerde(t); + ComplexMetricSerde serde = getComplexMetricSerde(typeName); writeBytes(serde.toBytes(val), out); + } else { + throw new IAE("Unable to serialize type[%s]", type); } } } @@ -467,21 +470,24 @@ public static InputRow fromBytes( //Read metrics int metricSize = WritableUtils.readVInt(in); for (int i = 0; i < metricSize; i++) { - String metric = readString(in); - String type = getType(metric, aggs, i); - byte metricNullability = in.readByte(); + final String metric = readString(in); + final AggregatorFactory agg = getAggregator(metric, aggs, i); + final ValueType type = agg.getType(); + final String typeName = agg.getTypeName(); + final byte metricNullability = in.readByte(); + if (metricNullability == NullHandling.IS_NULL_BYTE) { // metric value is null. continue; } - if ("float".equals(type)) { + if (ValueType.FLOAT.equals(type)) { event.put(metric, in.readFloat()); - } else if ("long".equals(type)) { + } else if (ValueType.LONG.equals(type)) { event.put(metric, WritableUtils.readVLong(in)); - } else if ("double".equals(type)) { + } else if (ValueType.DOUBLE.equals(type)) { event.put(metric, in.readDouble()); } else { - ComplexMetricSerde serde = getComplexMetricSerde(type); + ComplexMetricSerde serde = getComplexMetricSerde(typeName); byte[] value = readBytes(in); event.put(metric, serde.fromBytes(value, 0, value.length)); } @@ -495,15 +501,15 @@ public static InputRow fromBytes( } @Nullable - private static String getType(String metric, AggregatorFactory[] aggs, int i) + private static AggregatorFactory getAggregator(String metric, AggregatorFactory[] aggs, int i) { if (aggs[i].getName().equals(metric)) { - return aggs[i].getTypeName(); + return aggs[i]; } log.warn("Aggs disordered, fall backs to loop."); for (AggregatorFactory agg : aggs) { if (agg.getName().equals(metric)) { - return agg.getTypeName(); + return agg; } } return null; diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java index 3eb03b6807ff..8b3b021cb723 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java @@ -37,6 +37,7 @@ import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Rule; @@ -104,12 +105,14 @@ public void testSerde() final AggregatorFactory mockedAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(mockedAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedAggregator); EasyMock.expect(mockedAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + EasyMock.expect(mockedAggregatorFactory.getType()).andReturn(ValueType.DOUBLE).anyTimes(); EasyMock.expect(mockedAggregatorFactory.getName()).andReturn("mockedAggregator").anyTimes(); final AggregatorFactory mockedNullAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(mockedNullAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedNullAggregator); EasyMock.expect(mockedNullAggregatorFactory.getName()).andReturn("mockedNullAggregator").anyTimes(); EasyMock.expect(mockedNullAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); + EasyMock.expect(mockedNullAggregatorFactory.getType()).andReturn(ValueType.DOUBLE).anyTimes(); EasyMock.replay(mockedAggregatorFactory, mockedNullAggregatorFactory); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 1d4bec4f99e1..144d4b9a88c2 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -104,7 +104,6 @@ import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ListIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; -import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.RealtimeTuningConfig; import org.apache.druid.segment.indexing.TuningConfig; @@ -1427,7 +1426,7 @@ Map getQueryableIndexMap() private static ColumnHolder createColumn(DimensionSchema dimensionSchema) { - return new TestColumn(IncrementalIndex.TYPE_MAP.get(dimensionSchema.getValueType())); + return new TestColumn(dimensionSchema.getValueType()); } private static ColumnHolder createColumn(AggregatorFactory aggregatorFactory) diff --git a/processing/src/main/java/org/apache/druid/query/DruidMetrics.java b/processing/src/main/java/org/apache/druid/query/DruidMetrics.java index b813c6b75d69..b1e49af02bf1 100644 --- a/processing/src/main/java/org/apache/druid/query/DruidMetrics.java +++ b/processing/src/main/java/org/apache/druid/query/DruidMetrics.java @@ -20,6 +20,7 @@ package org.apache.druid.query; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.segment.column.ValueType; import java.util.List; @@ -45,9 +46,7 @@ public static int findNumComplexAggs(List aggs) { int retVal = 0; for (AggregatorFactory agg : aggs) { - // This needs to change when we have support column types better - if (!"float".equals(agg.getTypeName()) && !"long".equals(agg.getTypeName()) && !"double" - .equals(agg.getTypeName())) { + if (ValueType.COMPLEX.equals(agg.getType())) { retVal++; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index 93f221771208..44b6a2ac1d8c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -25,6 +25,8 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.PerSegmentQueryOptimizationContext; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ValueTypes; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import javax.annotation.Nullable; @@ -214,19 +216,37 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * {@link #deserialize} and the type accepted by {@link #combine}. However, it is *not* necessarily the same type * returned by {@link #finalizeComputation}. * - * If the type is complex (i.e. not a simple, numeric {@link org.apache.druid.segment.column.ValueType}) then there + * If the type is complex (i.e. not a simple, numeric {@link ValueType}) then there * must be a corresponding {@link org.apache.druid.segment.serde.ComplexMetricSerde} which was registered with * {@link org.apache.druid.segment.serde.ComplexMetrics#registerSerde} using this type name. * - * If you need a ValueType enum corresponding to this aggregator, a good way to do that is: - * - * 
-   *   Optional.ofNullable(GuavaUtils.getEnumIfPresent(ValueType.class, aggregator.getTypeName()))
-   *           .orElse(ValueType.COMPLEX);
-   *
+ * If you need a ValueType enum corresponding to this aggregator, use {@link #getTypeName} instead. */ public abstract String getTypeName(); + /** + * Get the type name for the 'finalized' type for this aggregator, i.e. the type of the value returned by + * {@link #finalizeComputation}. This may be the same as or different than the types expected in {@link #deserialize} + * and {@link #combine}. + * + * If you need a ValueType enum corresponding to this aggregator, use {@link #getFinalizedType} instead. + */ + //public abstract String getFinalizedTypeName(); + public String getFinalizedTypeName() + { + return getTypeName(); + } + + public ValueType getType() + { + return ValueTypes.aggregatorTypeNameToType(getTypeName()); + } + + public ValueType getFinalizedType() + { + return ValueTypes.aggregatorTypeNameToType(getFinalizedTypeName()); + } + /** * Returns the maximum size that this aggregator will require in bytes for intermediate storage of results. * diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/BufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/BufferAggregator.java index 98608546ccc4..b047f0ee03c5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/BufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/BufferAggregator.java @@ -101,7 +101,7 @@ public interface BufferAggregator extends HotLoopCallee * Implementations must not change the position, limit or mark of the given buffer * * Implementations are only required to support this method if they are aggregations which - * have an {@link AggregatorFactory#getTypeName()} of "float". + * have an {@link AggregatorFactory#getType()} ()} of {@link org.apache.druid.segment.column.ValueType#FLOAT}. * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * * @param buf byte buffer storing the byte array representation of the aggregate @@ -118,7 +118,7 @@ public interface BufferAggregator extends HotLoopCallee * Implementations must not change the position, limit or mark of the given buffer * * Implementations are only required to support this method if they are aggregations which - * have an {@link AggregatorFactory#getTypeName()} of "long". + * have an {@link AggregatorFactory#getType()} of of {@link org.apache.druid.segment.column.ValueType#LONG}. * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * * @param buf byte buffer storing the byte array representation of the aggregate @@ -135,7 +135,7 @@ public interface BufferAggregator extends HotLoopCallee * Implementations must not change the position, limit or mark of the given buffer * * Implementations are only required to support this method if they are aggregations which - * have an {@link AggregatorFactory#getTypeName()} of "double". + * have an {@link AggregatorFactory#getType()} of of {@link org.apache.druid.segment.column.ValueType#DOUBLE}. * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * * The default implementation casts {@link BufferAggregator#getFloat(ByteBuffer, int)} to double. diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java index 599a2c492020..17ba8e59109d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import javax.annotation.Nullable; @@ -136,7 +137,7 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return "long"; + return ValueType.LONG.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java index f76cd3d51516..dbe4d9671b41 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java @@ -185,6 +185,12 @@ public String getTypeName() return delegate.getTypeName(); } + @Override + public String getFinalizedTypeName() + { + return delegate.getFinalizedTypeName(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java index c32348dfbb34..30bea8a543f4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java @@ -206,6 +206,12 @@ public String getTypeName() return "histogram"; } + @Override + public String getFinalizedTypeName() + { + return "histogramVisual"; + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java index 006034a0e6a7..5e6fe5f5c516 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -33,6 +33,7 @@ import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -269,7 +270,7 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return "float"; + return ValueType.FLOAT.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java index aecc95245c21..08eea82637c8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java @@ -21,6 +21,8 @@ import org.apache.druid.guice.annotations.ExtensionPoint; import org.apache.druid.java.util.common.Cacheable; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; import java.util.Comparator; @@ -43,6 +45,13 @@ public interface PostAggregator extends Cacheable @Nullable String getName(); + String getTypeName(); + + default ValueType getType() + { + return ValueTypes.aggregatorTypeNameToType(getTypeName()); + } + /** * Returns a richer post aggregator which are built from the given aggregators with their names and some accessible * environmental variables such as ones in the object scope. diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 941b937ee43c..937fd71b1d9e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -142,9 +142,9 @@ public Object deserialize(Object object) public String getTypeName() { if (storeDoubleAsFloat) { - return "float"; + return ValueType.FLOAT.toString(); } - return "double"; + return ValueType.DOUBLE.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java index f82f5d3ef36d..79b85ec5393b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -123,7 +123,7 @@ public Object deserialize(Object object) @Override public String getTypeName() { - return "float"; + return ValueType.FLOAT.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java index bbc7b43cd57e..9a07c88ca846 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java @@ -125,7 +125,7 @@ public Object deserialize(Object object) @Override public String getTypeName() { - return "long"; + return ValueType.LONG.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java index 35e7938c8b5c..e904e78204ed 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java @@ -142,6 +142,12 @@ public String getTypeName() return delegate.getTypeName(); } + @Override + public String getFinalizedTypeName() + { + return delegate.getFinalizedTypeName(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java index 99e96c556417..9687af69d64c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java @@ -33,6 +33,7 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -189,7 +190,7 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return storeDoubleAsFloat ? "float" : "double"; + return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java index 2ecfc46ba841..34a12dbac346 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java @@ -32,6 +32,7 @@ import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -187,7 +188,7 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return "float"; + return ValueType.FLOAT.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java index 6add01fb4dea..2d7ebb84af38 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java @@ -32,6 +32,7 @@ import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.NilColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -185,7 +186,7 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return "long"; + return ValueType.LONG.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java index 9e28b084bb61..e741aa98c141 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java @@ -30,6 +30,7 @@ import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Collections; @@ -149,7 +150,7 @@ public byte[] getCacheKey() @Override public String getTypeName() { - return "string"; + return ValueType.STRING.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java index 8b619153cd4a..bd9d601e2e8b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java @@ -43,6 +43,7 @@ import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.DimensionHandlerUtils; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -289,6 +290,12 @@ public String getTypeName() return "hyperUnique"; } + @Override + public String getFinalizedTypeName() + { + return round ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 0920ee3e7bfc..6e5e84faf195 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -36,6 +36,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -276,7 +277,15 @@ public byte[] getCacheKey() public String getTypeName() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? "float" : "double"; + return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + } + + @Override + public String getFinalizedTypeName() + { + // this is a copy of getTypeName in the hopes that someday groupby v1 is no more and it will report it's actual + // type + return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 74d10fa516d2..5655c4040de5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -36,6 +36,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -273,7 +274,13 @@ public byte[] getCacheKey() public String getTypeName() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return "float"; + return ValueType.FLOAT.toString(); + } + + @Override + public String getFinalizedTypeName() + { + return ValueType.FLOAT.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 5b7e16dec0bc..bc2c21f16132 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -36,6 +36,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -271,7 +272,13 @@ public byte[] getCacheKey() public String getTypeName() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return "long"; + return ValueType.LONG.toString(); + } + + @Override + public String getFinalizedTypeName() + { + return ValueType.LONG.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index ada369829e7b..5c1d1034f8da 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -36,6 +36,7 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -262,6 +263,12 @@ public String getTypeName() return "serializablePairLongString"; } + @Override + public String getFinalizedTypeName() + { + return ValueType.STRING.toString(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java index 7ee605ed682a..fc2136290faf 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java @@ -28,7 +28,9 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueTypes; +import javax.annotation.Nullable; import java.util.Comparator; import java.util.Map; import java.util.Set; @@ -37,17 +39,12 @@ */ public class HyperUniqueFinalizingPostAggregator implements PostAggregator { - private static final Comparator DOUBLE_COMPARATOR = Ordering.from(new Comparator() - { - @Override - public int compare(Double lhs, Double rhs) - { - return Double.compare(lhs, rhs); - } - }).nullsFirst(); + private static final Comparator DOUBLE_COMPARATOR = + Ordering.from((Comparator) (lhs, rhs) -> Double.compare(lhs, rhs)).nullsFirst(); private final String name; private final String fieldName; + @Nullable private final AggregatorFactory aggregatorFactory; @JsonCreator @@ -62,7 +59,7 @@ public HyperUniqueFinalizingPostAggregator( private HyperUniqueFinalizingPostAggregator( String name, String fieldName, - AggregatorFactory aggregatorFactory + @Nullable AggregatorFactory aggregatorFactory ) { this.fieldName = Preconditions.checkNotNull(fieldName, "fieldName is null"); @@ -106,6 +103,14 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return aggregatorFactory != null + ? aggregatorFactory.getFinalizedTypeName() + : ValueTypes.defaultAggregationTypeName(); + } + @Override public HyperUniqueFinalizingPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java index b3ab8c3b4ef8..10bc3ad33e7f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java @@ -272,6 +272,12 @@ public String getTypeName() } } + @Override + public String getFinalizedTypeName() + { + return round ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index dd2503ed6e93..3769d955e30b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -38,6 +38,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -274,7 +275,15 @@ public byte[] getCacheKey() public String getTypeName() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? "float" : "double"; + return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + } + + @Override + public String getFinalizedTypeName() + { + // this is a copy of getTypeName in the hopes that someday groupby v1 is no more and it will report it's actual + // type + return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 78024f9c618f..e19899b7d30b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -38,6 +38,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -271,7 +272,13 @@ public byte[] getCacheKey() public String getTypeName() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return "float"; + return ValueType.FLOAT.toString(); + } + + @Override + public String getFinalizedTypeName() + { + return ValueType.FLOAT.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 3cc333f9e86e..6ba5ca531448 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -37,6 +37,7 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -269,7 +270,13 @@ public byte[] getCacheKey() public String getTypeName() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return "long"; + return ValueType.LONG.toString(); + } + + @Override + public String getFinalizedTypeName() + { + return ValueType.LONG.toString(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index e97f8b509781..649a7cf2fedd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -38,6 +38,7 @@ import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -219,6 +220,12 @@ public String getTypeName() return "serializablePairLongString"; } + @Override + public String getFinalizedTypeName() + { + return ValueType.STRING.name(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java index d1da4d3189f0..59230335e03f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java @@ -32,6 +32,7 @@ import org.apache.druid.query.aggregation.VectorAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import javax.annotation.Nullable; @@ -81,6 +82,12 @@ public String getTypeName() return "doubleMean"; } + @Override + public String getFinalizedTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java index 10c693c042a1..30425a16fedc 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.DoubleSumAggregator; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; @@ -133,6 +134,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public ArithmeticPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java index af8cd002e1ef..860e1e20aa30 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java @@ -26,6 +26,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.HashSet; @@ -74,6 +75,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return constantValue instanceof Long ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + } + @Override public ConstantPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 1e017b13c081..68cead8ea515 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.HashSet; @@ -99,6 +100,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public DoubleGreatestPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java index 1a143faed40f..98385486beaf 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.HashSet; @@ -99,6 +100,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public DoubleLeastPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java index 9184e14e6d01..a22703e23636 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java @@ -35,6 +35,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.utils.CollectionUtils; import javax.annotation.Nullable; @@ -60,7 +61,9 @@ public class ExpressionPostAggregator implements PostAggregator private final String name; private final String expression; private final Comparator comparator; + @Nullable private final String ordering; + private final ExprMacroTable macroTable; private final Map> finalizers; @@ -77,27 +80,13 @@ public ExpressionPostAggregator( @JsonProperty("ordering") @Nullable String ordering, @JacksonInject ExprMacroTable macroTable ) - { - this(name, expression, ordering, macroTable, ImmutableMap.of()); - } - - /** - * Constructor for {@link #decorate(Map)}. - */ - private ExpressionPostAggregator( - final String name, - final String expression, - @Nullable final String ordering, - final ExprMacroTable macroTable, - final Map> finalizers - ) { this( name, expression, ordering, macroTable, - finalizers, + ImmutableMap.of(), Suppliers.memoize(() -> Parser.parse(expression, macroTable)) ); } @@ -118,7 +107,8 @@ private ExpressionPostAggregator( macroTable, finalizers, parsed, - Suppliers.memoize(() -> parsed.get().analyzeInputs().getRequiredBindings())); + Suppliers.memoize(() -> parsed.get().analyzeInputs().getRequiredBindings()) + ); } private ExpressionPostAggregator( @@ -179,6 +169,14 @@ public String getName() return name; } + @Override + public String getTypeName() + { + // this is wrong, replace with Expr output type based on the input types once it is available + // but treat as string for now + return ValueType.STRING.toString(); + } + @Override public ExpressionPostAggregator decorate(final Map aggregators) { @@ -187,7 +185,7 @@ public ExpressionPostAggregator decorate(final Map ag expression, ordering, macroTable, - CollectionUtils.mapValues(aggregators, aggregatorFactory -> obj -> aggregatorFactory.finalizeComputation(obj)), + CollectionUtils.mapValues(aggregators, aggregatorFactory -> aggregatorFactory::finalizeComputation), parsed, dependentFields ); @@ -199,6 +197,7 @@ public String getExpression() return expression; } + @Nullable @JsonProperty("ordering") public String getOrdering() { @@ -230,7 +229,7 @@ public enum Ordering implements Comparator * Ensures the following order: numeric > NaN > Infinite. * * The name may be referenced via Ordering.valueOf(String) in the constructor {@link - * ExpressionPostAggregator#ExpressionPostAggregator(String, String, String, ExprMacroTable, Map)}. + * ExpressionPostAggregator#ExpressionPostAggregator(String, String, String, ExprMacroTable, Map, Supplier)}. */ @SuppressWarnings("unused") numericFirst { @@ -287,10 +286,6 @@ public boolean equals(Object o) @Override public int hashCode() { - int result = name != null ? name.hashCode() : 0; - result = 31 * result + expression.hashCode(); - result = 31 * result + comparator.hashCode(); - result = 31 * result + (ordering != null ? ordering.hashCode() : 0); - return result; + return Objects.hash(name, expression, comparator, ordering); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java index 17506d373098..88532bb2f1a6 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -26,9 +26,9 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; - import java.util.Comparator; import java.util.Map; import java.util.Set; @@ -40,16 +40,23 @@ public class FieldAccessPostAggregator implements PostAggregator @Nullable private final String name; private final String fieldName; + private final String typeName; @JsonCreator public FieldAccessPostAggregator( @JsonProperty("name") @Nullable String name, @JsonProperty("fieldName") String fieldName ) + { + this(name, fieldName, ValueTypes.defaultAggregationTypeName()); + } + + private FieldAccessPostAggregator(@Nullable String name, String fieldName, String typeName) { Preconditions.checkNotNull(fieldName); this.name = name; this.fieldName = fieldName; + this.typeName = typeName; } @Override @@ -78,10 +85,28 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return typeName; + } + @Override public FieldAccessPostAggregator decorate(Map aggregators) { - return this; + final String typeName; + + if (aggregators != null && aggregators.containsKey(fieldName)) { + typeName = aggregators.get(fieldName).getTypeName(); + } else { + typeName = ValueTypes.defaultAggregationTypeName(); + } + + return new FieldAccessPostAggregator( + name, + fieldName, + typeName + ); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java index 3cb5c3137099..f0699da05b84 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java @@ -26,6 +26,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.Map; @@ -36,6 +37,7 @@ public class FinalizingFieldAccessPostAggregator implements PostAggregator { private final String name; private final String fieldName; + private final String finalizedTypeName; private final Comparator comparator; private final Function finalizer; @@ -45,18 +47,20 @@ public FinalizingFieldAccessPostAggregator( @JsonProperty("fieldName") String fieldName ) { - this(name, fieldName, null, null); + this(name, fieldName, null, null, null); } private FinalizingFieldAccessPostAggregator( final String name, final String fieldName, + final String finalizedTypeName, final Comparator comparator, final Function finalizer ) { this.name = name; this.fieldName = fieldName; + this.finalizedTypeName = finalizedTypeName; this.comparator = comparator; this.finalizer = finalizer; } @@ -94,29 +98,35 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return finalizedTypeName; + } + @Override public FinalizingFieldAccessPostAggregator decorate(final Map aggregators) { final Comparator theComparator; final Function theFinalizer; + final String finalizedTypeName; if (aggregators != null && aggregators.containsKey(fieldName)) { //noinspection unchecked theComparator = aggregators.get(fieldName).getComparator(); + theFinalizer = aggregators.get(fieldName)::finalizeComputation; + finalizedTypeName = aggregators.get(fieldName).getFinalizedTypeName(); } else { //noinspection unchecked theComparator = (Comparator) Comparators.naturalNullsFirst(); - } - - if (aggregators != null && aggregators.containsKey(fieldName)) { - theFinalizer = aggregators.get(fieldName)::finalizeComputation; - } else { theFinalizer = Function.identity(); + finalizedTypeName = ValueType.DOUBLE.toString(); } return new FinalizingFieldAccessPostAggregator( name, fieldName, + finalizedTypeName, theComparator, theFinalizer ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java index a80774e9d38e..3659b5e78d0b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.DoubleSumAggregator; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.mozilla.javascript.Context; @@ -174,6 +175,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.DOUBLE.toString(); + } + @Override public JavaScriptPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java index 89ca17103cea..7b878b7e0222 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.HashSet; @@ -99,6 +100,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.LONG.toString(); + } + @Override public LongGreatestPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java index 3ff47e923037..d4dcb75b48b3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.HashSet; @@ -99,6 +100,12 @@ public String getName() return name; } + @Override + public String getTypeName() + { + return ValueType.LONG.toString(); + } + @Override public LongLeastPostAggregator decorate(Map aggregators) { diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GrouperBufferComparatorUtils.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GrouperBufferComparatorUtils.java index d9c19552de4f..f8d8488d7824 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GrouperBufferComparatorUtils.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/GrouperBufferComparatorUtils.java @@ -147,13 +147,12 @@ public static Grouper.BufferComparator bufferComparatorWithAggregators( int aggIndex = OrderByColumnSpec.getAggIndexForOrderBy(orderSpec, Arrays.asList(aggregatorFactories)); if (aggIndex >= 0) { final StringComparator stringComparator = orderSpec.getDimensionComparator(); - final String typeName = aggregatorFactories[aggIndex].getTypeName(); + final ValueType valueType = aggregatorFactories[aggIndex].getType(); final int aggOffset = aggregatorOffsets[aggIndex] - Integer.BYTES; aggCount++; - final ValueType valueType = ValueType.fromString(typeName); - if (!ValueType.isNumeric(valueType)) { + if (!valueType.isNumeric()) { throw new IAE("Cannot order by a non-numeric aggregator[%s]", orderSpec); } diff --git a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index a4fd44068fac..d21e609984e2 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -860,8 +860,7 @@ private Comparator> objectComparatorWithAggs() fieldIndices.add(aggIndex); needsReverses.add(needsReverse); aggFlags.add(true); - final String typeName = aggregatorFactories[aggIndex].getTypeName(); - isNumericField.add(ValueType.isNumeric(ValueType.fromString(typeName))); + isNumericField.add(aggregatorFactories[aggIndex].getType().isNumeric()); comparators.add(orderSpec.getDimensionComparator()); } } diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java index 49175a06873b..d3e698a3c456 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -27,6 +27,7 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.DictionaryEncodedColumn; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ValueTypes; import org.apache.druid.segment.data.ReadableOffset; import javax.annotation.Nullable; @@ -118,7 +119,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi ValueType type = columnHolder.getCapabilities().getType(); if (type.isNumeric()) { - return type.makeNumericWrappingDimensionSelector(makeColumnValueSelector(dimension), extractionFn); + return ValueTypes.makeNumericWrappingDimensionSelector(type, makeColumnValueSelector(dimension), extractionFn); } final DictionaryEncodedColumn column = getCachedColumn(dimension, DictionaryEncodedColumn.class); diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index 767f45fdc138..2c55f51c9fa5 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -23,10 +23,8 @@ import com.google.common.collect.ImmutableList; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; -import org.apache.druid.common.guava.GuavaUtils; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.dimension.DimensionSpec; @@ -206,7 +204,7 @@ private Builder() public Builder add(final String columnName, @Nullable final ValueType columnType) { // Name must be nonnull, but type can be null (if the type is unknown) - Preconditions.checkNotNull(columnName, "'columnName' must be nonnull"); + Preconditions.checkNotNull(columnName, "'columnName' must be non-null"); columnTypeList.add(Pair.of(columnName, columnType)); return this; } @@ -237,21 +235,14 @@ public Builder addDimensions(final List dimensions) public Builder addAggregators(final List aggregators) { for (final AggregatorFactory aggregator : aggregators) { - final ValueType type = GuavaUtils.getEnumIfPresent( - ValueType.class, - StringUtils.toUpperCase(aggregator.getTypeName()) - ); - - // Use null instead of COMPLEX for nonnumeric types, since in that case, the type depends on whether or not - // the aggregator is finalized, and we don't know (a) if it will be finalized, or even (b) what the type would - // be if it were finalized. So null (i.e. unknown) is the proper thing to do. - // - // Another note: technically, we don't know what the finalized type will be even if the type here is numeric, - // but we're assuming that it doesn't change upon finalization. All builtin aggregators work this way. + final ValueType type = aggregator.getType(); - if (type != null && type.isNumeric()) { + // Use null instead of COMPLEX for non-primitive and non-scalar types, since in that case, the type depends on + // whether or not the aggregator is finalized, and we don't know if it will be finalized + if (type.isPrimitiveScalar() && type.equals(aggregator.getFinalizedType())) { add(aggregator.getName(), type); } else { + // either complex type or finalized to a different type add(aggregator.getName(), null); } } @@ -268,8 +259,8 @@ public Builder addPostAggregators(final List postAggregators) "postAggregators must have nonnull names" ); - // PostAggregators don't have known types; use null for the type. - add(name, null); + // unlike aggregators, the type we see here is what we get, no further finalization will occur + add(name, postAggregator.getType()); } return this; diff --git a/processing/src/main/java/org/apache/druid/segment/column/SimpleColumnHolder.java b/processing/src/main/java/org/apache/druid/segment/column/SimpleColumnHolder.java index a340d4f3ab50..8b7162c37f75 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/SimpleColumnHolder.java +++ b/processing/src/main/java/org/apache/druid/segment/column/SimpleColumnHolder.java @@ -85,6 +85,6 @@ public SpatialIndex getSpatialIndex() @Override public SettableColumnValueSelector makeNewSettableColumnValueSelector() { - return getCapabilities().getType().makeNewSettableColumnValueSelector(); + return ValueTypes.makeNewSettableColumnValueSelector(getCapabilities().getType()); } } diff --git a/processing/src/main/java/org/apache/druid/segment/column/ValueType.java b/processing/src/main/java/org/apache/druid/segment/column/ValueType.java deleted file mode 100644 index 4bdbcfb87fe0..000000000000 --- a/processing/src/main/java/org/apache/druid/segment/column/ValueType.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment.column; - -import com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.extraction.ExtractionFn; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.DimensionSelector; -import org.apache.druid.segment.DoubleWrappingDimensionSelector; -import org.apache.druid.segment.FloatWrappingDimensionSelector; -import org.apache.druid.segment.LongWrappingDimensionSelector; -import org.apache.druid.segment.selector.settable.SettableColumnValueSelector; -import org.apache.druid.segment.selector.settable.SettableDimensionValueSelector; -import org.apache.druid.segment.selector.settable.SettableDoubleColumnValueSelector; -import org.apache.druid.segment.selector.settable.SettableFloatColumnValueSelector; -import org.apache.druid.segment.selector.settable.SettableLongColumnValueSelector; -import org.apache.druid.segment.selector.settable.SettableObjectColumnValueSelector; - -import javax.annotation.Nullable; - -/** - * Should be the same as {@link org.apache.druid.data.input.impl.DimensionSchema.ValueType}. - * TODO merge them when druid-api is merged back into the main repo - */ -public enum ValueType -{ - FLOAT { - @Override - public DimensionSelector makeNumericWrappingDimensionSelector( - ColumnValueSelector numericColumnValueSelector, - @Nullable ExtractionFn extractionFn - ) - { - return new FloatWrappingDimensionSelector(numericColumnValueSelector, extractionFn); - } - - @Override - public SettableColumnValueSelector makeNewSettableColumnValueSelector() - { - return new SettableFloatColumnValueSelector(); - } - }, - DOUBLE { - @Override - public DimensionSelector makeNumericWrappingDimensionSelector( - ColumnValueSelector numericColumnValueSelector, - @Nullable ExtractionFn extractionFn - ) - { - return new DoubleWrappingDimensionSelector(numericColumnValueSelector, extractionFn); - } - - @Override - public SettableColumnValueSelector makeNewSettableColumnValueSelector() - { - return new SettableDoubleColumnValueSelector(); - } - }, - LONG { - @Override - public DimensionSelector makeNumericWrappingDimensionSelector( - ColumnValueSelector numericColumnValueSelector, - @Nullable ExtractionFn extractionFn - ) - { - return new LongWrappingDimensionSelector(numericColumnValueSelector, extractionFn); - } - - @Override - public SettableColumnValueSelector makeNewSettableColumnValueSelector() - { - return new SettableLongColumnValueSelector(); - } - }, - STRING { - @Override - public SettableColumnValueSelector makeNewSettableColumnValueSelector() - { - return new SettableDimensionValueSelector(); - } - }, - COMPLEX { - @Override - public SettableColumnValueSelector makeNewSettableColumnValueSelector() - { - return new SettableObjectColumnValueSelector(); - } - }; - - public DimensionSelector makeNumericWrappingDimensionSelector( - ColumnValueSelector numericColumnValueSelector, - @Nullable ExtractionFn extractionFn - ) - { - throw new UnsupportedOperationException("Not a numeric value type: " + name()); - } - - public abstract SettableColumnValueSelector makeNewSettableColumnValueSelector(); - - public boolean isNumeric() - { - return isNumeric(this); - } - - @JsonCreator - @Nullable - public static ValueType fromString(@Nullable String name) - { - if (name == null) { - return null; - } - return valueOf(StringUtils.toUpperCase(name)); - } - - public static boolean isNumeric(ValueType type) - { - if (type == ValueType.LONG || type == ValueType.FLOAT || type == ValueType.DOUBLE) { - return true; - } - return false; - } -} diff --git a/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java b/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java new file mode 100644 index 000000000000..8328f611dbc6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.column; + +import org.apache.druid.common.guava.GuavaUtils; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.extraction.ExtractionFn; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.DimensionSelector; +import org.apache.druid.segment.DoubleWrappingDimensionSelector; +import org.apache.druid.segment.FloatWrappingDimensionSelector; +import org.apache.druid.segment.LongWrappingDimensionSelector; +import org.apache.druid.segment.selector.settable.SettableColumnValueSelector; +import org.apache.druid.segment.selector.settable.SettableDimensionValueSelector; +import org.apache.druid.segment.selector.settable.SettableDoubleColumnValueSelector; +import org.apache.druid.segment.selector.settable.SettableFloatColumnValueSelector; +import org.apache.druid.segment.selector.settable.SettableLongColumnValueSelector; +import org.apache.druid.segment.selector.settable.SettableObjectColumnValueSelector; + +import javax.annotation.Nullable; + +public class ValueTypes +{ + public static SettableColumnValueSelector makeNewSettableColumnValueSelector(ValueType valueType) + { + switch (valueType) { + case DOUBLE: + return new SettableDoubleColumnValueSelector(); + case FLOAT: + return new SettableFloatColumnValueSelector(); + case LONG: + return new SettableLongColumnValueSelector(); + case STRING: + return new SettableDimensionValueSelector(); + default: + return new SettableObjectColumnValueSelector<>(); + } + } + + public static DimensionSelector makeNumericWrappingDimensionSelector( + ValueType valueType, + ColumnValueSelector numericColumnValueSelector, + @Nullable ExtractionFn extractionFn + ) + { + switch (valueType) { + case DOUBLE: + return new DoubleWrappingDimensionSelector(numericColumnValueSelector, extractionFn); + case FLOAT: + return new FloatWrappingDimensionSelector(numericColumnValueSelector, extractionFn); + case LONG: + return new LongWrappingDimensionSelector(numericColumnValueSelector, extractionFn); + default: + throw new UnsupportedOperationException("Not a numeric value type: " + valueType.name()); + } + } + + public static ValueType aggregatorTypeNameToType(String name) + { + ValueType v = GuavaUtils.getEnumIfPresent( + ValueType.class, + StringUtils.toUpperCase(name) + ); + + return v == null ? ValueType.COMPLEX : v; + } + + public static String defaultAggregationTypeName() + { + return ValueType.DOUBLE.toString(); + } + + private ValueTypes() + { + // no instantiation + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java index 82b92766d5ef..a0afdac7dad8 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java @@ -24,7 +24,6 @@ import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -32,7 +31,6 @@ import com.google.errorprone.annotations.concurrent.GuardedBy; import org.apache.druid.collections.NonBlockingPool; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.common.guava.GuavaUtils; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.MapBasedRow; import org.apache.druid.data.input.Row; @@ -42,7 +40,6 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -103,19 +100,6 @@ public abstract class IncrementalIndex extends AbstractIndex imp { private volatile DateTime maxIngestedEventTime; - // Used to discover ValueType based on the class of values in a row - // Also used to convert between the duplicate ValueType enums in DimensionSchema (druid-api) and main druid. - public static final Map TYPE_MAP = ImmutableMap.builder() - .put(Long.class, ValueType.LONG) - .put(Double.class, ValueType.DOUBLE) - .put(Float.class, ValueType.FLOAT) - .put(String.class, ValueType.STRING) - .put(DimensionSchema.ValueType.LONG, ValueType.LONG) - .put(DimensionSchema.ValueType.FLOAT, ValueType.FLOAT) - .put(DimensionSchema.ValueType.STRING, ValueType.STRING) - .put(DimensionSchema.ValueType.DOUBLE, ValueType.DOUBLE) - .build(); - /** * Column selector used at ingestion time for inputs to aggregators. * @@ -145,9 +129,7 @@ class IncrementalIndexInputRowColumnSelectorFactory implements ColumnSelectorFac public ColumnValueSelector makeColumnValueSelector(final String column) { final String typeName = agg.getTypeName(); - final boolean isComplexMetric = - GuavaUtils.getEnumIfPresent(ValueType.class, StringUtils.toUpperCase(typeName)) == null || - typeName.equalsIgnoreCase(ValueType.COMPLEX.name()); + final boolean isComplexMetric = ValueType.COMPLEX.equals(agg.getType()); final ColumnValueSelector selector = baseSelectorFactory.makeColumnValueSelector(column); @@ -308,7 +290,7 @@ protected IncrementalIndex( this.dimensionDescsList = new ArrayList<>(); for (DimensionSchema dimSchema : dimensionsSpec.getDimensions()) { - ValueType type = TYPE_MAP.get(dimSchema.getValueType()); + ValueType type = dimSchema.getValueType(); String dimName = dimSchema.getName(); ColumnCapabilitiesImpl capabilities = makeCapabilitiesFromValueType(type); capabilities.setHasBitmapIndexes(dimSchema.hasBitmapIndex()); @@ -1116,21 +1098,19 @@ public MetricDesc(int index, AggregatorFactory factory) { this.index = index; this.name = factory.getName(); + this.capabilities = new ColumnCapabilitiesImpl().setIsComplete(true); + capabilities.setType(factory.getType()); String typeInfo = factory.getTypeName(); - this.capabilities = new ColumnCapabilitiesImpl().setIsComplete(true); - if ("float".equalsIgnoreCase(typeInfo)) { - capabilities.setType(ValueType.FLOAT); - this.type = typeInfo; - } else if ("long".equalsIgnoreCase(typeInfo)) { - capabilities.setType(ValueType.LONG); - this.type = typeInfo; - } else if ("double".equalsIgnoreCase(typeInfo)) { - capabilities.setType(ValueType.DOUBLE); + if (factory.getType().isPrimitiveScalar()) { this.type = typeInfo; } else { - capabilities.setType(ValueType.COMPLEX); - this.type = ComplexMetrics.getSerdeForType(typeInfo).getTypeName(); + ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeInfo); + if (serde != null) { + this.type = serde.getTypeName(); + } else { + throw new ISE("Don't know how to handle type[%s]", typeInfo); + } } } diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java index 69c71641c858..4773871a3e11 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; @@ -85,7 +86,8 @@ private DimensionSelector makeDimensionSelectorUndecorated(DimensionSpec dimensi return DimensionSelector.constant(null, extractionFn); } if (capabilities.getType().isNumeric()) { - return capabilities.getType().makeNumericWrappingDimensionSelector( + return ValueTypes.makeNumericWrappingDimensionSelector( + capabilities.getType(), makeColumnValueSelector(dimension), extractionFn ); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java index cace5c236770..fd9cef060c3b 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java @@ -20,6 +20,32 @@ package org.apache.druid.query.aggregation; import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.js.JavaScriptConfig; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.any.DoubleAnyAggregatorFactory; +import org.apache.druid.query.aggregation.any.FloatAnyAggregatorFactory; +import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; +import org.apache.druid.query.aggregation.any.StringAnyAggregatorFactory; +import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; +import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.mean.DoubleMeanAggregatorFactory; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.filter.SelectorDimFilter; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; @@ -28,7 +54,7 @@ /** */ -public class AggregatorFactoryTest +public class AggregatorFactoryTest extends InitializedNullHandlingTest { @Test @@ -77,4 +103,168 @@ public void testMergeAggregators() Assert.assertNull(AggregatorFactory.mergeAggregators(ImmutableList.of(af1, af2)) ); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new JavaScriptAggregatorFactory( + "js", + ImmutableList.of("col"), + "function(a,b) { return a + b; }", + "function() { return 0; }", + "function(a,b) { return a + b }", + new JavaScriptConfig(true) + ), + // long aggs + new LongSumAggregatorFactory("longSum", "long-col"), + new LongMinAggregatorFactory("longMin", "long-col"), + new LongMaxAggregatorFactory("longMax", "long-col"), + new LongFirstAggregatorFactory("longFirst", "long-col"), + new LongLastAggregatorFactory("longLast", "long-col"), + new LongAnyAggregatorFactory("longAny", "long-col"), + // double aggs + new DoubleSumAggregatorFactory("doubleSum", "double-col"), + new DoubleMinAggregatorFactory("doubleMin", "double-col"), + new DoubleMaxAggregatorFactory("doubleMax", "double-col"), + new DoubleFirstAggregatorFactory("doubleFirst", "double-col"), + new DoubleLastAggregatorFactory("doubleLast", "double-col"), + new DoubleAnyAggregatorFactory("doubleAny", "double-col"), + new DoubleMeanAggregatorFactory("doubleMean", "double-col"), + // float aggs + new FloatSumAggregatorFactory("floatSum", "float-col"), + new FloatMinAggregatorFactory("floatMin", "float-col"), + new FloatMaxAggregatorFactory("floatMax", "float-col"), + new FloatFirstAggregatorFactory("floatFirst", "float-col"), + new FloatLastAggregatorFactory("floatLast", "float-col"), + new FloatAnyAggregatorFactory("floatAny", "float-col"), + // string aggregators + new StringFirstAggregatorFactory("stringFirst", "col", 1024), + new StringLastAggregatorFactory("stringLast", "col", 1024), + new StringAnyAggregatorFactory("stringAny", "col", 1024), + // sketch aggs + new CardinalityAggregatorFactory("cardinality", ImmutableList.of(DefaultDimensionSpec.of("some-col")), false), + new HyperUniquesAggregatorFactory("hyperUnique", "hyperunique"), + new HistogramAggregatorFactory("histogram", "histogram", ImmutableList.of(0.25f, 0.5f, 0.75f)), + // delegate aggs + new FilteredAggregatorFactory( + new HyperUniquesAggregatorFactory("filtered", "hyperunique"), + new SelectorDimFilter("col", "hello", null) + ), + new SuppressedAggregatorFactory( + new HyperUniquesAggregatorFactory("suppressed", "hyperunique") + ) + ) + .postAggregators( + new FinalizingFieldAccessPostAggregator("count-finalize", "count"), + new FinalizingFieldAccessPostAggregator("js-finalize", "js"), + // long aggs + new FinalizingFieldAccessPostAggregator("longSum-finalize", "longSum"), + new FinalizingFieldAccessPostAggregator("longMin-finalize", "longMin"), + new FinalizingFieldAccessPostAggregator("longMax-finalize", "longMax"), + new FinalizingFieldAccessPostAggregator("longFirst-finalize", "longFirst"), + new FinalizingFieldAccessPostAggregator("longLast-finalize", "longLast"), + new FinalizingFieldAccessPostAggregator("longAny-finalize", "longAny"), + // double + new FinalizingFieldAccessPostAggregator("doubleSum-finalize", "doubleSum"), + new FinalizingFieldAccessPostAggregator("doubleMin-finalize", "doubleMin"), + new FinalizingFieldAccessPostAggregator("doubleMax-finalize", "doubleMax"), + new FinalizingFieldAccessPostAggregator("doubleFirst-finalize", "doubleFirst"), + new FinalizingFieldAccessPostAggregator("doubleLast-finalize", "doubleLast"), + new FinalizingFieldAccessPostAggregator("doubleAny-finalize", "doubleAny"), + new FinalizingFieldAccessPostAggregator("doubleMean-finalize", "doubleMean"), + // finalized floats + new FinalizingFieldAccessPostAggregator("floatSum-finalize", "floatSum"), + new FinalizingFieldAccessPostAggregator("floatMin-finalize", "floatMin"), + new FinalizingFieldAccessPostAggregator("floatMax-finalize", "floatMax"), + new FinalizingFieldAccessPostAggregator("floatFirst-finalize", "floatFirst"), + new FinalizingFieldAccessPostAggregator("floatLast-finalize", "floatLast"), + new FinalizingFieldAccessPostAggregator("floatAny-finalize", "floatAny"), + // finalized strings + new FinalizingFieldAccessPostAggregator("stringFirst-finalize", "stringFirst"), + new FinalizingFieldAccessPostAggregator("stringLast-finalize", "stringLast"), + new FinalizingFieldAccessPostAggregator("stringAny-finalize", "stringAny"), + // finalized sketch + new FinalizingFieldAccessPostAggregator("cardinality-finalize", "cardinality"), + new FinalizingFieldAccessPostAggregator("hyperUnique-finalize", "hyperUnique"), + new FinalizingFieldAccessPostAggregator("histogram-finalize", "histogram"), + // finalized delegate + new FinalizingFieldAccessPostAggregator("filtered-finalize", "filtered"), + new FinalizingFieldAccessPostAggregator("suppressed-finalize", "suppressed") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + // aggs + .add("count", ValueType.LONG) + .add("js", ValueType.FLOAT) + .add("longSum", ValueType.LONG) + .add("longMin", ValueType.LONG) + .add("longMax", ValueType.LONG) + .add("longFirst", ValueType.LONG) + .add("longLast", ValueType.LONG) + .add("longAny", ValueType.LONG) + .add("doubleSum", ValueType.DOUBLE) + .add("doubleMin", ValueType.DOUBLE) + .add("doubleMax", ValueType.DOUBLE) + .add("doubleFirst", ValueType.DOUBLE) + .add("doubleLast", ValueType.DOUBLE) + .add("doubleAny", ValueType.DOUBLE) + .add("doubleMean", null) + .add("floatSum", ValueType.FLOAT) + .add("floatMin", ValueType.FLOAT) + .add("floatMax", ValueType.FLOAT) + .add("floatFirst", ValueType.FLOAT) + .add("floatLast", ValueType.FLOAT) + .add("floatAny", ValueType.FLOAT) + .add("stringFirst", null) + .add("stringLast", null) + .add("stringAny", ValueType.STRING) + .add("cardinality", null) + .add("hyperUnique", null) + .add("histogram", null) + .add("filtered", null) + .add("suppressed", null) + // postaggs + .add("count-finalize", ValueType.LONG) + .add("js-finalize", ValueType.FLOAT) + .add("longSum-finalize", ValueType.LONG) + .add("longMin-finalize", ValueType.LONG) + .add("longMax-finalize", ValueType.LONG) + .add("longFirst-finalize", ValueType.LONG) + .add("longLast-finalize", ValueType.LONG) + .add("longAny-finalize", ValueType.LONG) + .add("doubleSum-finalize", ValueType.DOUBLE) + .add("doubleMin-finalize", ValueType.DOUBLE) + .add("doubleMax-finalize", ValueType.DOUBLE) + .add("doubleFirst-finalize", ValueType.DOUBLE) + .add("doubleLast-finalize", ValueType.DOUBLE) + .add("doubleAny-finalize", ValueType.DOUBLE) + .add("doubleMean-finalize", ValueType.DOUBLE) + .add("floatSum-finalize", ValueType.FLOAT) + .add("floatMin-finalize", ValueType.FLOAT) + .add("floatMax-finalize", ValueType.FLOAT) + .add("floatFirst-finalize", ValueType.FLOAT) + .add("floatLast-finalize", ValueType.FLOAT) + .add("floatAny-finalize", ValueType.FLOAT) + .add("stringFirst-finalize", ValueType.STRING) + .add("stringLast-finalize", ValueType.STRING) + .add("stringAny-finalize", ValueType.STRING) + .add("cardinality-finalize", ValueType.DOUBLE) + .add("hyperUnique-finalize", ValueType.DOUBLE) + .add("histogram-finalize", ValueType.COMPLEX) + .add("filtered-finalize", ValueType.DOUBLE) + .add("suppressed-finalize", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregatorTest.java index eb8b0a7a415f..8b7b085dec43 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregatorTest.java @@ -23,8 +23,15 @@ import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import org.apache.druid.hll.HyperLogLogCollector; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.hamcrest.CoreMatchers; import org.junit.Assert; import org.junit.Test; @@ -87,4 +94,36 @@ public void testComputeRounded() Assert.assertThat(cardinality, CoreMatchers.instanceOf(Long.class)); Assert.assertEquals(99L, cardinality); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new HyperUniquesAggregatorFactory("approxCount", "col"), + new HyperUniquesAggregatorFactory("approxCountRound", "col", false, true) + ) + .postAggregators( + new HyperUniqueFinalizingPostAggregator("a", "approxCount"), + new HyperUniqueFinalizingPostAggregator("b", "approxCountRound") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("approxCount", null) + .add("approxCountRound", null) + .add("a", ValueType.DOUBLE) + .add("b", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java index c0f7a19895f3..d581a1103e1e 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java @@ -23,9 +23,17 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.expression.TestExprMacroTable; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; @@ -221,4 +229,39 @@ public void testNumericFirstOrdering() Assert.assertTrue(numericFirst.compare(Double.POSITIVE_INFINITY, Double.NaN) < 0); Assert.assertTrue(numericFirst.compare(Double.NEGATIVE_INFINITY, Double.NaN) < 0); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new LongSumAggregatorFactory("sum", "col"), + new CountAggregatorFactory("count") + ) + .postAggregators( + new ArithmeticPostAggregator( + "avg", + "/", + ImmutableList.of( + new FieldAccessPostAggregator("_count", "count"), + new FieldAccessPostAggregator("_sum", "sum") + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("sum", ValueType.LONG) + .add("count", ValueType.LONG) + .add("avg", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/ConstantPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/ConstantPostAggregatorTest.java index deac5bdbb36c..12be9be56d62 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/ConstantPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/ConstantPostAggregatorTest.java @@ -20,6 +20,13 @@ package org.apache.druid.query.aggregation.post; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -64,4 +71,34 @@ public void testSerde() throws Exception ); Assert.assertEquals(aggregator, aggregator1); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new ConstantPostAggregator("a", 3L), + new ConstantPostAggregator("b", 1.0f), + new ConstantPostAggregator("c", 5.0) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.LONG) + .add("b", ValueType.DOUBLE) + .add("c", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregatorTest.java index 689bde5c6d00..68f264ad5c27 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregatorTest.java @@ -19,9 +19,17 @@ package org.apache.druid.query.aggregation.post; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -93,4 +101,37 @@ public void testComparator() Assert.assertEquals(0, comp.compare(after, after)); Assert.assertEquals(1, comp.compare(after, before)); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new DoubleGreatestPostAggregator( + "a", + ImmutableList.of( + new ConstantPostAggregator("_a", 3L), + new ConstantPostAggregator("_b", 1.0f), + new ConstantPostAggregator("_c", 5.0) + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregatorTest.java index b729d30fbe01..3fc69ea4f4a2 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregatorTest.java @@ -19,9 +19,17 @@ package org.apache.druid.query.aggregation.post; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -93,4 +101,37 @@ public void testComparator() Assert.assertEquals(0, comp.compare(after, after)); Assert.assertEquals(1, comp.compare(after, before)); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new DoubleLeastPostAggregator( + "a", + ImmutableList.of( + new ConstantPostAggregator("_a", 3L), + new ConstantPostAggregator("_b", 1.0f), + new ConstantPostAggregator("_c", 5.0) + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java index 662219d1572f..71f36fd33800 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java @@ -19,7 +19,16 @@ package org.apache.druid.query.aggregation.post; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -48,4 +57,38 @@ public void testCompute() metricValues.put(aggName, agg.get()); Assert.assertEquals(new Long(3L), fieldAccessPostAggregator.compute(metricValues)); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new DoubleSumAggregatorFactory("double", "col1"), + new FloatSumAggregatorFactory("float", "col2") + ) + .postAggregators( + new FieldAccessPostAggregator("a", "count"), + new FieldAccessPostAggregator("b", "double"), + new FieldAccessPostAggregator("c", "float") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("double", ValueType.DOUBLE) + .add("float", ValueType.FLOAT) + .add("a", ValueType.LONG) + .add("b", ValueType.DOUBLE) + .add("c", ValueType.FLOAT) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java index 1c027be7e2f8..bd92e7e7337a 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java @@ -27,15 +27,22 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.AggregationTestHelper; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.CountAggregator; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; import org.apache.druid.query.groupby.GroupByQueryRunnerTest; import org.apache.druid.query.groupby.ResultRow; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Rule; @@ -50,7 +57,7 @@ import java.util.List; import java.util.Map; -public class FinalizingFieldAccessPostAggregatorTest +public class FinalizingFieldAccessPostAggregatorTest extends InitializedNullHandlingTest { @Rule public final TemporaryFolder tempFoler = new TemporaryFolder(); @@ -78,6 +85,7 @@ public void testComputedWithFinalizing() AggregatorFactory aggFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(aggFactory.getComparator()).andReturn(Comparators.naturalNullsFirst()).once(); EasyMock.expect(aggFactory.finalizeComputation("test")).andReturn(3L).once(); + EasyMock.expect(aggFactory.getFinalizedTypeName()).andReturn(ValueType.LONG.toString()).once(); EasyMock.replay(aggFactory); FinalizingFieldAccessPostAggregator postAgg = buildDecorated( @@ -101,6 +109,7 @@ public void testComputedInArithmeticPostAggregator() AggregatorFactory aggFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(aggFactory.getComparator()).andReturn(Comparators.naturalNullsFirst()).once(); EasyMock.expect(aggFactory.finalizeComputation("test")).andReturn(3L).once(); + EasyMock.expect(aggFactory.getFinalizedTypeName()).andReturn(ValueType.LONG.toString()).once(); EasyMock.replay(aggFactory); FinalizingFieldAccessPostAggregator postAgg = buildDecorated( @@ -139,6 +148,8 @@ public void testComparatorsWithFinalizing() EasyMock.expect(aggFactory.getComparator()) .andReturn(Ordering.natural().nullsLast()) .times(1); + + EasyMock.expect(aggFactory.getFinalizedTypeName()).andReturn(ValueType.LONG.toString()).once(); EasyMock.replay(aggFactory); FinalizingFieldAccessPostAggregator postAgg = buildDecorated( @@ -266,6 +277,36 @@ public void testSerde() throws IOException ); } + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new StringFirstAggregatorFactory("stringo", "col", 1024) + ) + .postAggregators( + new FieldAccessPostAggregator("a", "stringo"), + new FinalizingFieldAccessPostAggregator("b", "stringo") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("stringo", null) + .add("a", ValueType.COMPLEX) + .add("b", ValueType.STRING) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } + private static FinalizingFieldAccessPostAggregator buildDecorated( String name, String fieldName, diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java index 63e4978cb7dc..fa38dec72c5a 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregatorTest.java @@ -20,7 +20,14 @@ package org.apache.druid.query.aggregation.post; import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.js.JavaScriptConfig; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -31,6 +38,7 @@ public class JavaScriptPostAggregatorTest { + private static final String ABS_PERCENT_FUNCTION = "function(delta, total) { return 100 * Math.abs(delta) / total; }"; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -43,12 +51,10 @@ public void testCompute() metricValues.put("delta", -10.0); metricValues.put("total", 100.0); - - String absPercentFunction = "function(delta, total) { return 100 * Math.abs(delta) / total; }"; javaScriptPostAggregator = new JavaScriptPostAggregator( "absPercent", Lists.newArrayList("delta", "total"), - absPercentFunction, + ABS_PERCENT_FUNCTION, JavaScriptConfig.getEnabledInstance() ); @@ -58,11 +64,10 @@ public void testCompute() @Test public void testComputeJavaScriptNotAllowed() { - String absPercentFunction = "function(delta, total) { return 100 * Math.abs(delta) / total; }"; JavaScriptPostAggregator aggregator = new JavaScriptPostAggregator( "absPercent", Lists.newArrayList("delta", "total"), - absPercentFunction, + ABS_PERCENT_FUNCTION, new JavaScriptConfig(false) ); @@ -70,4 +75,37 @@ public void testComputeJavaScriptNotAllowed() expectedException.expectMessage("JavaScript is disabled"); aggregator.compute(new HashMap<>()); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new LongSumAggregatorFactory("total", "total"), + new LongSumAggregatorFactory("delta", "delta") + ) + .postAggregators( + new JavaScriptPostAggregator( + "a", + Lists.newArrayList("delta", "total"), + ABS_PERCENT_FUNCTION, + JavaScriptConfig.getEnabledInstance() + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("total", ValueType.LONG) + .add("delta", ValueType.LONG) + .add("a", ValueType.DOUBLE) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregatorTest.java index c3c1d11ab974..304978c069a7 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregatorTest.java @@ -19,9 +19,17 @@ package org.apache.druid.query.aggregation.post; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -93,4 +101,37 @@ public void testComparator() Assert.assertEquals(0, comp.compare(after, after)); Assert.assertEquals(1, comp.compare(after, before)); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new LongGreatestPostAggregator( + "a", + ImmutableList.of( + new ConstantPostAggregator("_a", 3L), + new ConstantPostAggregator("_b", 1.0f), + new ConstantPostAggregator("_c", 5.0) + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregatorTest.java index c889ddf70c20..bc30588926dc 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregatorTest.java @@ -19,9 +19,17 @@ package org.apache.druid.query.aggregation.post; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregator; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -93,4 +101,37 @@ public void testComparator() Assert.assertEquals(0, comp.compare(after, after)); Assert.assertEquals(1, comp.compare(after, before)); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count") + ) + .postAggregators( + new LongLeastPostAggregator( + "a", + ImmutableList.of( + new ConstantPostAggregator("_a", 3L), + new ConstantPostAggregator("_b", 1.0f), + new ConstantPostAggregator("_c", 5.0) + ) + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("a", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java index 53d2b97696d4..358523d7aeee 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByLimitPushDownInsufficientBufferTest.java @@ -75,6 +75,7 @@ import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.SegmentId; import org.junit.After; import org.junit.Assert; @@ -93,7 +94,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -public class GroupByLimitPushDownInsufficientBufferTest +public class GroupByLimitPushDownInsufficientBufferTest extends InitializedNullHandlingTest { public static final ObjectMapper JSON_MAPPER; diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java index f3e0befbe9c8..31763671f834 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java @@ -684,7 +684,7 @@ public void testResultArraySignatureAllGran() .add("rows", ValueType.LONG) .add("index", ValueType.DOUBLE) .add("uniques", null) - .add("const", null) + .add("const", ValueType.LONG) .build(), new GroupByQueryQueryToolChest(null, null).resultArraySignature(query) ); @@ -709,7 +709,7 @@ public void testResultArraySignatureDayGran() .add("rows", ValueType.LONG) .add("index", ValueType.DOUBLE) .add("uniques", null) - .add("const", null) + .add("const", ValueType.LONG) .build(), new GroupByQueryQueryToolChest(null, null).resultArraySignature(query) ); diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java index 3f6e59b7f493..8b13462e69ae 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java @@ -382,7 +382,7 @@ public void testResultArraySignature() .add("rows", ValueType.LONG) .add("index", ValueType.DOUBLE) .add("uniques", null) - .add("const", null) + .add("const", ValueType.LONG) .build(), TOOL_CHEST.resultArraySignature(query) ); diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java index 8246f97b0e26..18e7f361712d 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java @@ -319,7 +319,7 @@ public void testResultArraySignature() .add("rows", ValueType.LONG) .add("index", ValueType.DOUBLE) .add("uniques", null) - .add("const", null) + .add("const", ValueType.LONG) .build(), new TopNQueryQueryToolChest(null, null).resultArraySignature(query) ); From a27a4c8004bc391fdde7baf64dbc9e2375167741 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 7 Apr 2020 04:07:14 -0700 Subject: [PATCH 02/22] more javadoc --- .../apache/druid/query/aggregation/AggregatorFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index 44b6a2ac1d8c..4d06d2045b6c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -231,17 +231,22 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * * If you need a ValueType enum corresponding to this aggregator, use {@link #getFinalizedType} instead. */ - //public abstract String getFinalizedTypeName(); public String getFinalizedTypeName() { return getTypeName(); } + /** + * Get the "intermediate" {@link ValueType} for this aggregator. See {@link #getTypeName}. + */ public ValueType getType() { return ValueTypes.aggregatorTypeNameToType(getTypeName()); } + /** + * Get the "finalized" {@link ValueType} for this aggregator. See {@link #getFinalizedTypeName} + */ public ValueType getFinalizedType() { return ValueTypes.aggregatorTypeNameToType(getFinalizedTypeName()); From f2aa12f64a8291e256ac58e039837057f289822e Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 8 Apr 2020 12:39:52 -0700 Subject: [PATCH 03/22] adjustments --- .../MomentSketchAggregatorFactoryTest.java | 4 +- .../TDigestSketchAggregatorFactoryTest.java | 2 +- .../theta/SketchAggregatorFactoryTest.java | 4 +- .../BloomFilterAggregatorFactoryTest.java | 4 +- ...ApproximateHistogramAggregatorFactory.java | 3 +- ...ixedBucketsHistogramAggregatorFactory.java | 2 +- .../ApproximateHistogramAggregatorTest.java | 42 +++++++++++++++++++ ...pproximateHistogramPostAggregatorTest.java | 2 +- ...dBucketsHistogramBufferAggregatorTest.java | 42 +++++++++++++++++++ .../druid/segment/column/RowSignature.java | 8 ++-- .../aggregation/AggregatorFactoryTest.java | 2 +- 11 files changed, 99 insertions(+), 16 deletions(-) diff --git a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java index 5a1991a7a62b..7b69d06054d9 100644 --- a/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java +++ b/extensions-contrib/momentsketch/src/test/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactoryTest.java @@ -76,8 +76,8 @@ public void testResultArraySignature() RowSignature.builder() .addTimeColumn() .add("count", ValueType.LONG) - .add("moment", null) - .add("momentMerge", null) + .add("moment", ValueType.COMPLEX) + .add("momentMerge", ValueType.COMPLEX) .add("moment-access", ValueType.COMPLEX) .add("moment-finalize", ValueType.COMPLEX) .add("momentMerge-access", ValueType.COMPLEX) diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java index 82e482530e75..cb08f67c875a 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactoryTest.java @@ -55,7 +55,7 @@ public void testResultArraySignature() RowSignature.builder() .addTimeColumn() .add("count", ValueType.LONG) - .add("tdigest", null) + .add("tdigest", ValueType.COMPLEX) .add("tdigest-access", ValueType.COMPLEX) .add("tdigest-finalize", ValueType.COMPLEX) .build(), diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java index 9299fbe8f6e3..af31183e43aa 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java @@ -70,9 +70,9 @@ public void testResultArraySignature() .addTimeColumn() .add("count", ValueType.LONG) .add("oldBuild", null) - .add("oldMerge", null) + .add("oldMerge", ValueType.COMPLEX) .add("oldMergeFinalize", null) - .add("merge", null) + .add("merge", ValueType.COMPLEX) .add("mergeFinalize", null) .add("oldBuild-access", ValueType.COMPLEX) .add("oldBuild-finalize", ValueType.DOUBLE) diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java index f03b12eb2767..dc9d0b924541 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactoryTest.java @@ -59,8 +59,8 @@ public void testResultArraySignature() RowSignature.builder() .addTimeColumn() .add("count", ValueType.LONG) - .add("bloom", null) - .add("bloomMerge", null) + .add("bloom", ValueType.COMPLEX) + .add("bloomMerge", ValueType.COMPLEX) .add("bloom-access", ValueType.COMPLEX) .add("bloom-finalize", ValueType.COMPLEX) .add("bloomMerge-access", ValueType.COMPLEX) diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index 3567f39b9913..e82c1e24d3ef 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -35,7 +35,6 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -308,7 +307,7 @@ public String getTypeName() @Override public String getFinalizedTypeName() { - return finalizeAsBase64Binary ? ValueType.STRING.toString() : "histogram"; + return finalizeAsBase64Binary ? getTypeName() : "histogram"; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java index 25fd40165a9c..418c4581ed0a 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java @@ -255,7 +255,7 @@ public String getTypeName() @Override public String getFinalizedTypeName() { - return ValueType.STRING.toString(); + return finalizeAsBase64Binary ? getTypeName() : ValueType.STRING.toString(); } @Override diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java index fd41e150214b..5c8bfd1d9fcd 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorTest.java @@ -20,8 +20,16 @@ package org.apache.druid.query.aggregation.histogram; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.TestFloatColumnSelector; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; @@ -113,4 +121,38 @@ public void testFinalize() throws Exception finalStringBinary ); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new ApproximateHistogramAggregatorFactory("approxHisto", "col", null, null, null, null, false), + new ApproximateHistogramAggregatorFactory("approxHistoBin", "col", null, null, null, null, true) + ) + .postAggregators( + new FieldAccessPostAggregator("approxHisto-access", "approxHisto"), + new FinalizingFieldAccessPostAggregator("approxHisto-finalize", "approxHisto"), + new FieldAccessPostAggregator("approxHistoBin-access", "approxHistoBin"), + new FinalizingFieldAccessPostAggregator("approxHistoBin-finalize", "approxHistoBin") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("approxHisto", ValueType.COMPLEX) + .add("approxHistoBin", ValueType.COMPLEX) + .add("approxHisto-access", ValueType.COMPLEX) + .add("approxHisto-finalize", ValueType.COMPLEX) + .add("approxHistoBin-access", ValueType.COMPLEX) + .add("approxHistoBin-finalize", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java index 68823dcead81..d6af24b929e8 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramPostAggregatorTest.java @@ -93,7 +93,7 @@ public void testResultArraySignature() Assert.assertEquals( RowSignature.builder() .addTimeColumn() - .add("approxHisto", null) + .add("approxHisto", ValueType.COMPLEX) .add("bucket", ValueType.COMPLEX) .add("equal", ValueType.COMPLEX) .add("custom", ValueType.COMPLEX) diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramBufferAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramBufferAggregatorTest.java index aff00d88b672..dece34597b74 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramBufferAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramBufferAggregatorTest.java @@ -20,8 +20,16 @@ package org.apache.druid.query.aggregation.histogram; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.TestFloatColumnSelector; +import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -134,4 +142,38 @@ public void testFinalize() throws Exception finalStringBinary ); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new FixedBucketsHistogramAggregatorFactory("fixedHisto", "col", null, 0, 100, null, false), + new FixedBucketsHistogramAggregatorFactory("fixedHistoBin", "col", null, 0, 100, null, true) + ) + .postAggregators( + new FieldAccessPostAggregator("fixedHisto-access", "fixedHisto"), + new FinalizingFieldAccessPostAggregator("fixedHisto-finalize", "fixedHisto"), + new FieldAccessPostAggregator("fixedHistoBin-access", "fixedHistoBin"), + new FinalizingFieldAccessPostAggregator("fixedHistoBin-finalize", "fixedHistoBin") + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("fixedHisto", null) + .add("fixedHistoBin", ValueType.COMPLEX) + .add("fixedHisto-access", ValueType.COMPLEX) + .add("fixedHisto-finalize", ValueType.STRING) + .add("fixedHistoBin-access", ValueType.COMPLEX) + .add("fixedHistoBin-finalize", ValueType.COMPLEX) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index 2c55f51c9fa5..d2dde2cdc02f 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -237,12 +237,12 @@ public Builder addAggregators(final List aggregators) for (final AggregatorFactory aggregator : aggregators) { final ValueType type = aggregator.getType(); - // Use null instead of COMPLEX for non-primitive and non-scalar types, since in that case, the type depends on - // whether or not the aggregator is finalized, and we don't know if it will be finalized - if (type.isPrimitiveScalar() && type.equals(aggregator.getFinalizedType())) { + if (type.equals(aggregator.getFinalizedType())) { add(aggregator.getName(), type); } else { - // either complex type or finalized to a different type + // Use null if the type depends on whether or not the aggregator is finalized, since + // we don't know if it will be finalized or not. So null (i.e. unknown) is the proper + // thing to do. add(aggregator.getName(), null); } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java index fd9cef060c3b..3aa9aa3ffeb8 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java @@ -230,7 +230,7 @@ public void testResultArraySignature() .add("stringAny", ValueType.STRING) .add("cardinality", null) .add("hyperUnique", null) - .add("histogram", null) + .add("histogram", ValueType.COMPLEX) .add("filtered", null) .add("suppressed", null) // postaggs From c81ba6a1002864e56314ffeecb480957b2a2073d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sun, 19 Apr 2020 14:08:43 -0700 Subject: [PATCH 04/22] transition to getTypeName to be used exclusively for complex types --- .../DistinctCountAggregatorFactory.java | 12 ++++++ .../MomentSketchAggregatorFactory.java | 9 ++++ .../MomentSketchMaxPostAggregator.java | 4 +- .../MomentSketchMinPostAggregator.java | 4 +- .../MomentSketchQuantilePostAggregator.java | 4 +- .../movingaverage/AveragerFactoryWrapper.java | 6 +++ .../TDigestSketchAggregatorFactory.java | 10 +++++ ...TDigestSketchToQuantilePostAggregator.java | 4 +- ...DigestSketchToQuantilesPostAggregator.java | 4 +- .../TimestampAggregatorFactory.java | 11 +++-- .../hll/HllSketchAggregatorFactory.java | 13 +++++- .../HllSketchToEstimatePostAggregator.java | 4 +- ...tchToEstimateWithBoundsPostAggregator.java | 4 +- .../hll/HllSketchToStringPostAggregator.java | 4 +- .../hll/HllSketchUnionPostAggregator.java | 8 +++- .../DoublesSketchAggregatorFactory.java | 13 +++++- .../DoublesSketchToCDFPostAggregator.java | 4 +- ...oublesSketchToHistogramPostAggregator.java | 8 +++- ...DoublesSketchToQuantilePostAggregator.java | 4 +- ...oublesSketchToQuantilesPostAggregator.java | 4 +- .../DoublesSketchToRankPostAggregator.java | 4 +- .../DoublesSketchToStringPostAggregator.java | 4 +- .../theta/SketchConstantPostAggregator.java | 8 +++- .../theta/SketchEstimatePostAggregator.java | 4 +- .../theta/SketchMergeAggregatorFactory.java | 32 ++++++++++---- .../theta/SketchSetPostAggregator.java | 8 +++- .../theta/SketchToStringPostAggregator.java | 4 +- ...ArrayOfDoublesSketchAggregatorFactory.java | 13 +++++- ...rayOfDoublesSketchSetOpPostAggregator.java | 8 +++- ...rayOfDoublesSketchTTestPostAggregator.java | 4 +- ...etchToEstimateAndBoundsPostAggregator.java | 4 +- ...DoublesSketchToEstimatePostAggregator.java | 4 +- ...yOfDoublesSketchToMeansPostAggregator.java | 4 +- ...ublesSketchToNumEntriesPostAggregator.java | 4 +- ...SketchToQuantilesSketchPostAggregator.java | 8 +++- ...OfDoublesSketchToStringPostAggregator.java | 4 +- ...oublesSketchToVariancesPostAggregator.java | 4 +- .../bloom/BloomFilterAggregatorFactory.java | 9 ++++ ...ApproximateHistogramAggregatorFactory.java | 17 +++++++- .../histogram/BucketsPostAggregator.java | 8 +++- .../CustomBucketsPostAggregator.java | 8 +++- .../histogram/EqualBucketsPostAggregator.java | 8 +++- ...ixedBucketsHistogramAggregatorFactory.java | 16 ++++++- .../histogram/MaxPostAggregator.java | 4 +- .../histogram/MinPostAggregator.java | 4 +- .../histogram/QuantilePostAggregator.java | 4 +- .../histogram/QuantilesPostAggregator.java | 8 +++- .../PvaluefromZscorePostAggregator.java | 4 +- .../teststats/ZtestPostAggregator.java | 4 +- .../StandardDeviationPostAggregator.java | 4 +- .../variance/VarianceAggregatorFactory.java | 13 +++++- .../apache/druid/indexer/InputRowSerde.java | 6 +-- .../druid/indexer/InputRowSerdeTest.java | 2 - .../common/task/CompactionTaskTest.java | 2 +- .../query/aggregation/AggregatorFactory.java | 42 ++++++++----------- .../aggregation/CountAggregatorFactory.java | 4 +- .../FilteredAggregatorFactory.java | 11 ++++- .../HistogramAggregatorFactory.java | 17 +++++++- .../JavaScriptAggregatorFactory.java | 4 +- .../query/aggregation/PostAggregator.java | 8 +--- .../SimpleDoubleAggregatorFactory.java | 6 +-- .../SimpleFloatAggregatorFactory.java | 4 +- .../SimpleLongAggregatorFactory.java | 5 ++- .../SuppressedAggregatorFactory.java | 11 ++++- .../any/DoubleAnyAggregatorFactory.java | 4 +- .../any/FloatAnyAggregatorFactory.java | 4 +- .../any/LongAnyAggregatorFactory.java | 4 +- .../any/StringAnyAggregatorFactory.java | 4 +- .../CardinalityAggregatorFactory.java | 13 +++++- .../first/DoubleFirstAggregatorFactory.java | 10 ++--- .../first/FloatFirstAggregatorFactory.java | 8 ++-- .../first/LongFirstAggregatorFactory.java | 8 ++-- .../first/StringFirstAggregatorFactory.java | 13 +++++- .../HyperUniqueFinalizingPostAggregator.java | 7 ++-- .../HyperUniquesAggregatorFactory.java | 13 +++++- .../last/DoubleLastAggregatorFactory.java | 8 ++-- .../last/FloatLastAggregatorFactory.java | 8 ++-- .../last/LongLastAggregatorFactory.java | 8 ++-- .../last/StringLastAggregatorFactory.java | 13 +++++- .../mean/DoubleMeanAggregatorFactory.java | 13 +++++- .../post/ArithmeticPostAggregator.java | 4 +- .../post/ConstantPostAggregator.java | 4 +- .../post/DoubleGreatestPostAggregator.java | 4 +- .../post/DoubleLeastPostAggregator.java | 4 +- .../post/ExpressionPostAggregator.java | 4 +- .../post/FieldAccessPostAggregator.java | 21 +++++----- .../FinalizingFieldAccessPostAggregator.java | 24 ++++++----- .../post/JavaScriptPostAggregator.java | 4 +- .../post/LongGreatestPostAggregator.java | 4 +- .../post/LongLeastPostAggregator.java | 4 +- .../druid/segment/column/ValueTypes.java | 4 +- .../segment/incremental/IncrementalIndex.java | 22 +++++----- ...nalizingFieldAccessPostAggregatorTest.java | 6 +-- 93 files changed, 479 insertions(+), 256 deletions(-) diff --git a/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java b/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java index b74aba1e24bd..e3b5d51d35a6 100644 --- a/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java @@ -34,6 +34,7 @@ import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.DimensionSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -199,6 +200,17 @@ public String getTypeName() return "distinctCount"; } + /** + * this aggregator only works on a single segment, so even though it stores a + * {@link org.apache.druid.collections.bitmap.MutableBitmap} while computing, this value never leaves the aggregator + * and {@link DistinctCountAggregator#get} returns an integer for the number of set bits in the bitmap. + */ + @Override + public ValueType getType() + { + return ValueType.LONG; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java index 951c532e4a7d..8d93f108b803 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java @@ -245,6 +245,15 @@ public String getTypeName() return TYPE_NAME; } + /** + * actual type is {@link MomentSketchWrapper} + */ + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java index d91e650564f2..d9c8c29f5e84 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMaxPostAggregator.java @@ -59,9 +59,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java index 96875203ad34..207e153745a9 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchMinPostAggregator.java @@ -58,9 +58,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java index 28a6af1f262c..74d9fcbbad31 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchQuantilePostAggregator.java @@ -63,9 +63,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @JsonProperty diff --git a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java index 42cfc3a4a6a0..c3253e46fef2 100644 --- a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java +++ b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java @@ -172,6 +172,12 @@ public String getTypeName() return ValueType.COMPLEX.name(); } + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + /** * Not implemented. Throws UnsupportedOperationException. */ diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java index dc93f75fc061..e0e73d881a25 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java @@ -34,6 +34,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -216,6 +217,15 @@ public String getTypeName() return TYPE_NAME; } + /** + * actual type is {@link MergingDigest} + */ + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java index 4a41efd8dbfd..63a7f853caed 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilePostAggregator.java @@ -69,9 +69,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java index ebd9021eafcc..122969df6834 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchToQuantilesPostAggregator.java @@ -69,9 +69,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @JsonProperty diff --git a/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java b/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java index d94d265508d1..21f792447610 100644 --- a/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java +++ b/extensions-contrib/time-min-max/src/main/java/org/apache/druid/query/aggregation/TimestampAggregatorFactory.java @@ -201,15 +201,18 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } + /** + * actual type is {@link DateTime} + */ @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { - return "dateTime"; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java index 6485c22354de..18d63a90f133 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactory.java @@ -169,10 +169,19 @@ public Class classOfObject() }; } + /** + * actual type is {@link HllSketch} + */ + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { - return round ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + return round ? ValueType.LONG : ValueType.DOUBLE; } @Nullable diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java index 9f09917dc3e2..0d6b5a141134 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java @@ -64,9 +64,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java index e87f2b1ddfd9..9ba5c27495cb 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimateWithBoundsPostAggregator.java @@ -70,9 +70,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java index a4d925d1a836..1df0d670e11a 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToStringPostAggregator.java @@ -81,9 +81,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.STRING.toString(); + return ValueType.STRING; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java index c9dc1c82ded9..509f77ef6f10 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchUnionPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Comparator; @@ -70,10 +71,13 @@ public String getName() return name; } + /** + * actual type is {@link HllSketch} + */ @Override - public String getTypeName() + public ValueType getType() { - return "hllSketch"; + return ValueType.COMPLEX; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index e02ac9704023..6227385b8764 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -253,10 +253,19 @@ public String getTypeName() return DoublesSketchModule.DOUBLES_SKETCH; } + /** + * actual type is {@link DoublesSketch} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.LONG; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java index ac29a6f28a7a..9426c6c577de 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToCDFPostAggregator.java @@ -73,9 +73,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java index f069a3d2c97b..00ff3d16bbd8 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -75,10 +76,13 @@ public String getName() return name; } + /** + * actual type is {@link DoublesSketch} + */ @Override - public String getTypeName() + public ValueType getType() { - return "doublesSketch"; + return ValueType.COMPLEX; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java index 3af11295a2c9..1d48bd12a369 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilePostAggregator.java @@ -60,9 +60,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java index 7f73b9455119..f36fe8e43bdb 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToQuantilesPostAggregator.java @@ -61,9 +61,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java index f795a24fe60c..499676ee5071 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToRankPostAggregator.java @@ -60,9 +60,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java index 659d24034cb7..591a4253c8b5 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToStringPostAggregator.java @@ -57,9 +57,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.STRING.toString(); + return ValueType.STRING; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java index a9aa253da423..1c9cce5d27c0 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchConstantPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Collections; import java.util.Comparator; @@ -77,10 +78,13 @@ public String getName() return name; } + /** + * actual type is {@link SketchHolder} + */ @Override - public String getTypeName() + public ValueType getType() { - return SketchModule.THETA_SKETCH; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java index 750708ae1067..fb27e6a96f4f 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java @@ -102,9 +102,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java index 3679e191dfb2..692069449e1e 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java @@ -34,6 +34,7 @@ public class SketchMergeAggregatorFactory extends SketchAggregatorFactory { private final boolean shouldFinalize; private final boolean isInputThetaSketch; + @Nullable private final Integer errorBoundsStdDev; @JsonCreator @@ -47,8 +48,8 @@ public SketchMergeAggregatorFactory( ) { super(name, fieldName, size, AggregatorUtil.SKETCH_MERGE_CACHE_TYPE_ID); - this.shouldFinalize = (shouldFinalize == null) ? true : shouldFinalize.booleanValue(); - this.isInputThetaSketch = (isInputThetaSketch == null) ? false : isInputThetaSketch.booleanValue(); + this.shouldFinalize = (shouldFinalize == null) ? true : shouldFinalize; + this.isInputThetaSketch = (isInputThetaSketch == null) ? false : isInputThetaSketch; this.errorBoundsStdDev = errorBoundsStdDev; } @@ -104,6 +105,7 @@ public boolean getIsInputThetaSketch() return isInputThetaSketch; } + @Nullable @JsonProperty public Integer getErrorBoundsStdDev() { @@ -148,16 +150,28 @@ public String getTypeName() } } + /** + * actual type is {@link SketchHolder} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - if (shouldFinalize) { - if (errorBoundsStdDev != null) { - return "estimateWithErrorBounds"; - } - return ValueType.DOUBLE.toString(); + return ValueType.COMPLEX; + } + + /** + * if {@link #shouldFinalize} is set, actual type is {@link SketchEstimateWithErrorBounds} if + * {@link #errorBoundsStdDev} is set. + * + * if {@link #shouldFinalize} is NOT set, type is {@link SketchHolder} + */ + @Override + public ValueType getFinalizedType() + { + if (shouldFinalize && errorBoundsStdDev == null) { + return ValueType.DOUBLE; } - return getTypeName(); + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java index f4c2d0a7d108..84a7533822b4 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Comparator; import java.util.LinkedHashSet; @@ -94,10 +95,13 @@ public String getName() return name; } + /** + * actual type is {@link SketchHolder} + */ @Override - public String getTypeName() + public ValueType getType() { - return SketchModule.THETA_SKETCH; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java index 77f523737510..179ef594776f 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchToStringPostAggregator.java @@ -80,9 +80,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.STRING.toString(); + return ValueType.STRING; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java index 8688a9363981..e33de66aab0f 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java @@ -302,10 +302,19 @@ public String getTypeName() return ArrayOfDoublesSketchModule.ARRAY_OF_DOUBLES_SKETCH_BUILD_AGG; } + /** + * actual type is {@link ArrayOfDoublesSketch} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java index 8aaf3c171037..9093e224ded1 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchSetOpPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Comparator; @@ -81,10 +82,13 @@ public ArrayOfDoublesSketch compute(final Map combinedAggregator return operation.apply(nominalEntries, numberOfValues, sketches); } + /** + * actual type is {@link ArrayOfDoublesSketch} + */ @Override - public String getTypeName() + public ValueType getType() { - return "arrayOfDoublesSketch"; + return ValueType.COMPLEX; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java index b696c7b10457..5e3925d2326b 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchTTestPostAggregator.java @@ -88,9 +88,9 @@ public double[] compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } private static SummaryStatistics[] getStats(final ArrayOfDoublesSketch sketch) diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java index 5ef115201444..2f757532f2f9 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimateAndBoundsPostAggregator.java @@ -70,9 +70,9 @@ public double[] compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java index 8ed74bf12ef0..2724529bbf42 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToEstimatePostAggregator.java @@ -53,9 +53,9 @@ public Double compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java index 9cbab054e05a..42eb1f598e05 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToMeansPostAggregator.java @@ -69,9 +69,9 @@ public double[] compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java index f2b862cf3ce6..6c0cb25f70a3 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToNumEntriesPostAggregator.java @@ -53,9 +53,9 @@ public Integer compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java index e48c6932f07b..3a74da07ea3b 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToQuantilesSketchPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.util.Comparator; @@ -82,10 +83,13 @@ public DoublesSketch compute(final Map combinedAggregators) return qs; } + /** + * actual type is {@link DoublesSketch} + */ @Override - public String getTypeName() + public ValueType getType() { - return "doublesSketch"; + return ValueType.COMPLEX; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java index eb0b1a48afbd..207058370d77 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToStringPostAggregator.java @@ -56,9 +56,9 @@ public String compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.STRING.toString(); + return ValueType.STRING; } @Override diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java index 10ece6af754a..3a9bbbc4c33b 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchToVariancesPostAggregator.java @@ -69,9 +69,9 @@ public double[] compute(final Map combinedAggregators) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE_ARRAY.toString(); + return ValueType.DOUBLE_ARRAY; } @Override diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java index 28fcaffbbf5e..27cee40a57e2 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java @@ -184,6 +184,15 @@ public String getTypeName() return BloomFilterSerializersModule.BLOOM_FILTER_TYPE_NAME; } + /** + * actual type is {@link ByteBuffer} containing {@link BloomKFilter} + */ + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index e82c1e24d3ef..b9bcced0aab1 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -35,6 +35,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -304,10 +305,22 @@ public String getTypeName() return "approximateHistogram"; } + /** + * actual type is {@link ApproximateHistogram} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return finalizeAsBase64Binary ? getTypeName() : "histogram"; + return ValueType.COMPLEX; + } + + /** + * actual type is {@link ApproximateHistogram} if {@link #finalizeAsBase64Binary} is set, else {@link Histogram} + */ + @Override + public ValueType getFinalizedType() + { + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java index b4a4dd75cfe7..4fb8101e2f26 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Map; import java.util.Set; @@ -67,10 +68,13 @@ public Object compute(Map values) return ah.toHistogram(bucketSize, offset); } + /** + * actual type is {@link Histogram} + */ @Override - public String getTypeName() + public ValueType getType() { - return "histogram"; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java index b71d6c29df1f..4873366e3b38 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java @@ -27,6 +27,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Map; @@ -63,10 +64,13 @@ public Object compute(Map values) return ah.toHistogram(breaks); } + /** + * actual type is {@link Histogram} + */ @Override - public String getTypeName() + public ValueType getType() { - return "histogram"; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java index 5e84e3c5e4eb..c939ed5663d9 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Map; import java.util.Set; @@ -64,10 +65,13 @@ public Object compute(Map values) return ah.toHistogram(numBuckets); } + /** + * actual type is {@link Histogram} + */ @Override - public String getTypeName() + public ValueType getType() { - return "histogram"; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java index 418c4581ed0a..d4f8b8d10776 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java @@ -252,10 +252,22 @@ public String getTypeName() return FixedBucketsHistogramAggregator.TYPE_NAME; } + /** + * actual type is {@link FixedBucketsHistogram} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return finalizeAsBase64Binary ? getTypeName() : ValueType.STRING.toString(); + return ValueType.COMPLEX; + } + + /** + * actual type is {@link FixedBucketsHistogram} if {@link #finalizeAsBase64Binary} is set + */ + @Override + public ValueType getFinalizedType() + { + return finalizeAsBase64Binary ? ValueType.COMPLEX : ValueType.STRING; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java index 8454c9ff7a1e..a601c847102a 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MaxPostAggregator.java @@ -74,9 +74,9 @@ public Object compute(Map values) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java index baadc4561432..8d6c8eb4e12b 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/MinPostAggregator.java @@ -76,9 +76,9 @@ public Object compute(Map values) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java index adfbd1085d98..cac97b1bb5fc 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilePostAggregator.java @@ -86,9 +86,9 @@ public Object compute(Map values) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java index 90314204e017..82898abc1483 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/QuantilesPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; @@ -92,10 +93,13 @@ public Object compute(Map values) throw new ISE("Unknown value type: " + val.getClass()); } + /** + * actual type is {@link Quantiles} + */ @Override - public String getTypeName() + public ValueType getType() { - return "quantiles"; + return ValueType.COMPLEX; } @Override diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java index ae4f65414708..85b4ab499b1a 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/PvaluefromZscorePostAggregator.java @@ -107,9 +107,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java index 73a22b35e093..51e4a3bc0196 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/teststats/ZtestPostAggregator.java @@ -116,9 +116,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java index cdd6950d4a7c..3c18e2c737d2 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/StandardDeviationPostAggregator.java @@ -86,9 +86,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java index 5079134a52d0..0a2f1f19d1bd 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java @@ -92,10 +92,19 @@ public String getTypeName() return VARIANCE_TYPE_NAME; } + /** + * actual type is {@link VarianceAggregatorCollector} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.DOUBLE; } @Override diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java index 1307c91f6baa..13624242da6d 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java @@ -341,7 +341,6 @@ public static SerializeResult toBytes( } final ValueType type = aggFactory.getType(); - final String typeName = aggFactory.getTypeName(); if (agg.isNull()) { out.writeByte(NullHandling.IS_NULL_BYTE); @@ -355,7 +354,7 @@ public static SerializeResult toBytes( out.writeDouble(agg.getDouble()); } else if (ValueType.COMPLEX.equals(type)) { Object val = agg.get(); - ComplexMetricSerde serde = getComplexMetricSerde(typeName); + ComplexMetricSerde serde = getComplexMetricSerde(aggFactory.getTypeName()); writeBytes(serde.toBytes(val), out); } else { throw new IAE("Unable to serialize type[%s]", type); @@ -473,7 +472,6 @@ public static InputRow fromBytes( final String metric = readString(in); final AggregatorFactory agg = getAggregator(metric, aggs, i); final ValueType type = agg.getType(); - final String typeName = agg.getTypeName(); final byte metricNullability = in.readByte(); if (metricNullability == NullHandling.IS_NULL_BYTE) { @@ -487,7 +485,7 @@ public static InputRow fromBytes( } else if (ValueType.DOUBLE.equals(type)) { event.put(metric, in.readDouble()); } else { - ComplexMetricSerde serde = getComplexMetricSerde(typeName); + ComplexMetricSerde serde = getComplexMetricSerde(agg.getTypeName()); byte[] value = readBytes(in); event.put(metric, serde.fromBytes(value, 0, value.length)); } diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java index 8b3b021cb723..ed92f7e93d99 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/InputRowSerdeTest.java @@ -104,14 +104,12 @@ public void testSerde() final AggregatorFactory mockedAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(mockedAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedAggregator); - EasyMock.expect(mockedAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); EasyMock.expect(mockedAggregatorFactory.getType()).andReturn(ValueType.DOUBLE).anyTimes(); EasyMock.expect(mockedAggregatorFactory.getName()).andReturn("mockedAggregator").anyTimes(); final AggregatorFactory mockedNullAggregatorFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(mockedNullAggregatorFactory.factorize(EasyMock.anyObject(ColumnSelectorFactory.class))).andReturn(mockedNullAggregator); EasyMock.expect(mockedNullAggregatorFactory.getName()).andReturn("mockedNullAggregator").anyTimes(); - EasyMock.expect(mockedNullAggregatorFactory.getTypeName()).andReturn("double").anyTimes(); EasyMock.expect(mockedNullAggregatorFactory.getType()).andReturn(ValueType.DOUBLE).anyTimes(); EasyMock.replay(mockedAggregatorFactory, mockedNullAggregatorFactory); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 144d4b9a88c2..0c148a008175 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -1431,7 +1431,7 @@ private static ColumnHolder createColumn(DimensionSchema dimensionSchema) private static ColumnHolder createColumn(AggregatorFactory aggregatorFactory) { - return new TestColumn(ValueType.fromString(aggregatorFactory.getTypeName())); + return new TestColumn(aggregatorFactory.getType()); } private static class TestColumn implements ColumnHolder diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index 4d06d2045b6c..e0b2d7d518fd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -21,12 +21,12 @@ import org.apache.druid.guice.annotations.ExtensionPoint; import org.apache.druid.java.util.common.Cacheable; +import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.UOE; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.PerSegmentQueryOptimizationContext; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.column.ValueTypes; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import javax.annotation.Nullable; @@ -212,44 +212,36 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre public abstract List requiredFields(); /** - * Get the type name of the intermediate type for this aggregator. This is the same as the type returned by + * Get the "intermediate" {@link ValueType} for this aggregator. This is the same as the type returned by * {@link #deserialize} and the type accepted by {@link #combine}. However, it is *not* necessarily the same type * returned by {@link #finalizeComputation}. - * - * If the type is complex (i.e. not a simple, numeric {@link ValueType}) then there - * must be a corresponding {@link org.apache.druid.segment.serde.ComplexMetricSerde} which was registered with - * {@link org.apache.druid.segment.serde.ComplexMetrics#registerSerde} using this type name. - * - * If you need a ValueType enum corresponding to this aggregator, use {@link #getTypeName} instead. */ - public abstract String getTypeName(); + public abstract ValueType getType(); /** - * Get the type name for the 'finalized' type for this aggregator, i.e. the type of the value returned by + * Get the type for the final form of this this aggregator, i.e. the type of the value returned by * {@link #finalizeComputation}. This may be the same as or different than the types expected in {@link #deserialize} * and {@link #combine}. - * - * If you need a ValueType enum corresponding to this aggregator, use {@link #getFinalizedType} instead. + * @return */ - public String getFinalizedTypeName() - { - return getTypeName(); - } - - /** - * Get the "intermediate" {@link ValueType} for this aggregator. See {@link #getTypeName}. - */ - public ValueType getType() + public ValueType getFinalizedType() { - return ValueTypes.aggregatorTypeNameToType(getTypeName()); + return getType(); } /** - * Get the "finalized" {@link ValueType} for this aggregator. See {@link #getFinalizedTypeName} + * Get the complex type name of the intermediate type for this aggregator. + * + * This should ONLY be implemented if the type is complex (i.e. not a simple, numeric {@link ValueType}), and there + * must be a corresponding {@link org.apache.druid.segment.serde.ComplexMetricSerde} which was registered with + * {@link org.apache.druid.segment.serde.ComplexMetrics#registerSerde} using this type name. + * + * If you need a ValueType enum corresponding to this aggregator, use {@link #getTypeName} instead. */ - public ValueType getFinalizedType() + @Nullable + public String getTypeName() { - return ValueTypes.aggregatorTypeNameToType(getFinalizedTypeName()); + throw new ISE("Complex type name not is not available for %s of type %s", getName(), getType()); } /** diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java index 17ba8e59109d..78539d46cd4b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java @@ -135,9 +135,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java index dbe4d9671b41..1a3203a6c323 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java @@ -31,6 +31,7 @@ import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.joda.time.Interval; @@ -186,9 +187,15 @@ public String getTypeName() } @Override - public String getFinalizedTypeName() + public ValueType getType() { - return delegate.getFinalizedTypeName(); + return delegate.getType(); + } + + @Override + public ValueType getFinalizedType() + { + return delegate.getFinalizedType(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java index 30bea8a543f4..0973cab38ef0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java @@ -25,6 +25,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -206,10 +207,22 @@ public String getTypeName() return "histogram"; } + /** + * actual type is {@link Histogram} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return "histogramVisual"; + return ValueType.COMPLEX; + } + + /** + * actual type is {@link HistogramVisual} + */ + @Override + public ValueType getFinalizedType() + { + return ValueType.COMPLEX; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java index 5e6fe5f5c516..74027d7c2661 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -268,9 +268,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java index 08eea82637c8..010e7cc5e10f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java @@ -22,7 +22,6 @@ import org.apache.druid.guice.annotations.ExtensionPoint; import org.apache.druid.java.util.common.Cacheable; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; import java.util.Comparator; @@ -45,12 +44,7 @@ public interface PostAggregator extends Cacheable @Nullable String getName(); - String getTypeName(); - - default ValueType getType() - { - return ValueTypes.aggregatorTypeNameToType(getTypeName()); - } + ValueType getType(); /** * Returns a richer post aggregator which are built from the given aggregators with their names and some accessible diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 937fd71b1d9e..6fdeee3c1f23 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -139,12 +139,12 @@ public Object deserialize(Object object) } @Override - public String getTypeName() + public ValueType getType() { if (storeDoubleAsFloat) { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java index 79b85ec5393b..ba008065193a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -121,9 +121,9 @@ public Object deserialize(Object object) } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java index 9a07c88ca846..6263441169e9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java @@ -122,10 +122,11 @@ public Object deserialize(Object object) return object; } + @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java index e904e78204ed..217dfdad25f1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java @@ -23,6 +23,7 @@ import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import javax.annotation.Nullable; @@ -143,9 +144,15 @@ public String getTypeName() } @Override - public String getFinalizedTypeName() + public ValueType getType() { - return delegate.getFinalizedTypeName(); + return delegate.getType(); + } + + @Override + public ValueType getFinalizedType() + { + return delegate.getFinalizedType(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java index 9687af69d64c..c8c31a3fc866 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java @@ -188,9 +188,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java index 34a12dbac346..727eea1b613b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java @@ -186,9 +186,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java index 2d7ebb84af38..ceb7d83568a6 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java @@ -184,9 +184,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java index e741aa98c141..7be9fb7cafc3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java @@ -148,9 +148,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.STRING.toString(); + return ValueType.STRING; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java index bd9d601e2e8b..aa119849d31e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java @@ -290,10 +290,19 @@ public String getTypeName() return "hyperUnique"; } + /** + * actual type is {@link HyperLogLogCollector} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return round ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return round ? ValueType.LONG : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 6e5e84faf195..2c7f07a9b4b8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -274,18 +274,18 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { // this is a copy of getTypeName in the hopes that someday groupby v1 is no more and it will report it's actual - // type - return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + // type of COMPLEX + return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 5655c4040de5..ab9211f1277c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -271,16 +271,16 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index bc2c21f16132..697663c38574 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -269,16 +269,16 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index 5c1d1034f8da..d055cad0410e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -263,10 +263,19 @@ public String getTypeName() return "serializablePairLongString"; } + /** + * actual type is {@link SerializablePairLongString} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return ValueType.STRING.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.STRING; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java index fc2136290faf..8529f43b07ff 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; @@ -104,11 +105,11 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { return aggregatorFactory != null - ? aggregatorFactory.getFinalizedTypeName() - : ValueTypes.defaultAggregationTypeName(); + ? aggregatorFactory.getFinalizedType() + : ValueTypes.defaultAggregationType(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java index 10bc3ad33e7f..2cdda5bcafdc 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java @@ -272,10 +272,19 @@ public String getTypeName() } } + /** + * actual type is {@link HyperLogLogCollector} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return round ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return round ? ValueType.LONG : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 3769d955e30b..9eccf27e10dd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -272,18 +272,18 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { // this is a copy of getTypeName in the hopes that someday groupby v1 is no more and it will report it's actual // type - return storeDoubleAsFloat ? ValueType.FLOAT.toString() : ValueType.DOUBLE.toString(); + return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index e19899b7d30b..01d7808c830b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -269,16 +269,16 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { - return ValueType.FLOAT.toString(); + return ValueType.FLOAT; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 6ba5ca531448..5a8964dc3da2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -267,16 +267,16 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public ValueType getType() { // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override - public String getFinalizedTypeName() + public ValueType getFinalizedType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index 649a7cf2fedd..1c0373eb35b4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -220,10 +220,19 @@ public String getTypeName() return "serializablePairLongString"; } + /** + * actual type is {@link SerializablePairLongString} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return ValueType.STRING.name(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.STRING; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java index 59230335e03f..8b80ee9c26da 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java @@ -82,10 +82,19 @@ public String getTypeName() return "doubleMean"; } + /** + * actual type is {@link DoubleMeanHolder} + */ @Override - public String getFinalizedTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.COMPLEX; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java index 30425a16fedc..fb53f616e4e3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -135,9 +135,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java index 860e1e20aa30..8651f7f749ec 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ConstantPostAggregator.java @@ -76,9 +76,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return constantValue instanceof Long ? ValueType.LONG.toString() : ValueType.DOUBLE.toString(); + return constantValue instanceof Long ? ValueType.LONG : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java index 68cead8ea515..78ef821d8ce2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleGreatestPostAggregator.java @@ -101,9 +101,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java index 98385486beaf..6321a5370c85 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/DoubleLeastPostAggregator.java @@ -101,9 +101,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java index a22703e23636..bd55a9f25283 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java @@ -170,11 +170,11 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { // this is wrong, replace with Expr output type based on the input types once it is available // but treat as string for now - return ValueType.STRING.toString(); + return ValueType.STRING; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java index 88532bb2f1a6..c2db45290cac 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -26,6 +26,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; @@ -40,7 +41,7 @@ public class FieldAccessPostAggregator implements PostAggregator @Nullable private final String name; private final String fieldName; - private final String typeName; + private final ValueType type; @JsonCreator public FieldAccessPostAggregator( @@ -48,15 +49,15 @@ public FieldAccessPostAggregator( @JsonProperty("fieldName") String fieldName ) { - this(name, fieldName, ValueTypes.defaultAggregationTypeName()); + this(name, fieldName, ValueTypes.defaultAggregationType()); } - private FieldAccessPostAggregator(@Nullable String name, String fieldName, String typeName) + private FieldAccessPostAggregator(@Nullable String name, String fieldName, ValueType type) { Preconditions.checkNotNull(fieldName); this.name = name; this.fieldName = fieldName; - this.typeName = typeName; + this.type = type; } @Override @@ -86,26 +87,26 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return typeName; + return type; } @Override public FieldAccessPostAggregator decorate(Map aggregators) { - final String typeName; + final ValueType type; if (aggregators != null && aggregators.containsKey(fieldName)) { - typeName = aggregators.get(fieldName).getTypeName(); + type = aggregators.get(fieldName).getType(); } else { - typeName = ValueTypes.defaultAggregationTypeName(); + type = ValueTypes.defaultAggregationType(); } return new FieldAccessPostAggregator( name, fieldName, - typeName + type ); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java index f0699da05b84..f370d50c3170 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java @@ -27,9 +27,11 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.column.ValueTypes; import java.util.Comparator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; @@ -37,7 +39,7 @@ public class FinalizingFieldAccessPostAggregator implements PostAggregator { private final String name; private final String fieldName; - private final String finalizedTypeName; + private final ValueType finalizedType; private final Comparator comparator; private final Function finalizer; @@ -53,14 +55,14 @@ public FinalizingFieldAccessPostAggregator( private FinalizingFieldAccessPostAggregator( final String name, final String fieldName, - final String finalizedTypeName, + final ValueType finalizedType, final Comparator comparator, final Function finalizer ) { this.name = name; this.fieldName = fieldName; - this.finalizedTypeName = finalizedTypeName; + this.finalizedType = finalizedType; this.comparator = comparator; this.finalizer = finalizer; } @@ -99,9 +101,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return finalizedTypeName; + return finalizedType; } @Override @@ -109,24 +111,24 @@ public FinalizingFieldAccessPostAggregator decorate(final Map theComparator; final Function theFinalizer; - final String finalizedTypeName; + final ValueType finalizedType; if (aggregators != null && aggregators.containsKey(fieldName)) { //noinspection unchecked theComparator = aggregators.get(fieldName).getComparator(); theFinalizer = aggregators.get(fieldName)::finalizeComputation; - finalizedTypeName = aggregators.get(fieldName).getFinalizedTypeName(); + finalizedType = aggregators.get(fieldName).getFinalizedType(); } else { //noinspection unchecked theComparator = (Comparator) Comparators.naturalNullsFirst(); theFinalizer = Function.identity(); - finalizedTypeName = ValueType.DOUBLE.toString(); + finalizedType = ValueTypes.defaultAggregationType(); } return new FinalizingFieldAccessPostAggregator( name, fieldName, - finalizedTypeName, + finalizedType, theComparator, theFinalizer ); @@ -167,10 +169,10 @@ public boolean equals(Object o) FinalizingFieldAccessPostAggregator that = (FinalizingFieldAccessPostAggregator) o; - if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) { + if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (name != null ? !name.equals(that.name) : that.name != null) { + if (!Objects.equals(name, that.name)) { return false; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java index 3659b5e78d0b..b19c59f83677 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/JavaScriptPostAggregator.java @@ -176,9 +176,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java index 7b878b7e0222..3e57c1690059 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongGreatestPostAggregator.java @@ -101,9 +101,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java index d4dcb75b48b3..5353e195bb71 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/LongLeastPostAggregator.java @@ -101,9 +101,9 @@ public String getName() } @Override - public String getTypeName() + public ValueType getType() { - return ValueType.LONG.toString(); + return ValueType.LONG; } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java b/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java index 8328f611dbc6..2285eb1b81dc 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java @@ -82,9 +82,9 @@ public static ValueType aggregatorTypeNameToType(String name) return v == null ? ValueType.COMPLEX : v; } - public static String defaultAggregationTypeName() + public static ValueType defaultAggregationType() { - return ValueType.DOUBLE.toString(); + return ValueType.DOUBLE; } private ValueTypes() diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java index 0a3f5405f969..a89f4cc7a1cb 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java @@ -128,7 +128,6 @@ class IncrementalIndexInputRowColumnSelectorFactory implements ColumnSelectorFac @Override public ColumnValueSelector makeColumnValueSelector(final String column) { - final String typeName = agg.getTypeName(); final boolean isComplexMetric = ValueType.COMPLEX.equals(agg.getType()); final ColumnValueSelector selector = baseSelectorFactory.makeColumnValueSelector(column); @@ -139,10 +138,10 @@ public ColumnValueSelector makeColumnValueSelector(final String column) // Wrap selector in a special one that uses ComplexMetricSerde to modify incoming objects. // For complex aggregators that read from multiple columns, we wrap all of them. This is not ideal but it // has worked so far. - - final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName); + final String complexTypeName = agg.getTypeName(); + final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(complexTypeName); if (serde == null) { - throw new ISE("Don't know how to handle type[%s]", typeName); + throw new ISE("Don't know how to handle type[%s]", complexTypeName); } final ComplexMetricExtractor extractor = serde.getExtractor(); @@ -1105,18 +1104,19 @@ public MetricDesc(int index, AggregatorFactory factory) this.index = index; this.name = factory.getName(); this.capabilities = new ColumnCapabilitiesImpl().setIsComplete(true); - capabilities.setType(factory.getType()); + ValueType valueType = factory.getType(); + capabilities.setType(valueType); - String typeInfo = factory.getTypeName(); - if (factory.getType().isPrimitiveScalar()) { - this.type = typeInfo; - } else { - ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeInfo); + if (valueType.isComplex()) { + String complexTypeName = factory.getTypeName(); + ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(complexTypeName); if (serde != null) { this.type = serde.getTypeName(); } else { - throw new ISE("Don't know how to handle type[%s]", typeInfo); + throw new ISE("Don't know how to handle type[%s]", complexTypeName); } + } else { + this.type = valueType.toString(); } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java index bd92e7e7337a..f684d1497918 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java @@ -85,7 +85,7 @@ public void testComputedWithFinalizing() AggregatorFactory aggFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(aggFactory.getComparator()).andReturn(Comparators.naturalNullsFirst()).once(); EasyMock.expect(aggFactory.finalizeComputation("test")).andReturn(3L).once(); - EasyMock.expect(aggFactory.getFinalizedTypeName()).andReturn(ValueType.LONG.toString()).once(); + EasyMock.expect(aggFactory.getFinalizedType()).andReturn(ValueType.LONG).once(); EasyMock.replay(aggFactory); FinalizingFieldAccessPostAggregator postAgg = buildDecorated( @@ -109,7 +109,7 @@ public void testComputedInArithmeticPostAggregator() AggregatorFactory aggFactory = EasyMock.createMock(AggregatorFactory.class); EasyMock.expect(aggFactory.getComparator()).andReturn(Comparators.naturalNullsFirst()).once(); EasyMock.expect(aggFactory.finalizeComputation("test")).andReturn(3L).once(); - EasyMock.expect(aggFactory.getFinalizedTypeName()).andReturn(ValueType.LONG.toString()).once(); + EasyMock.expect(aggFactory.getFinalizedType()).andReturn(ValueType.LONG).once(); EasyMock.replay(aggFactory); FinalizingFieldAccessPostAggregator postAgg = buildDecorated( @@ -149,7 +149,7 @@ public void testComparatorsWithFinalizing() .andReturn(Ordering.natural().nullsLast()) .times(1); - EasyMock.expect(aggFactory.getFinalizedTypeName()).andReturn(ValueType.LONG.toString()).once(); + EasyMock.expect(aggFactory.getFinalizedType()).andReturn(ValueType.LONG).once(); EasyMock.replay(aggFactory); FinalizingFieldAccessPostAggregator postAgg = buildDecorated( From 702b548ec41b41173f846308f092f7327e83c83d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sun, 19 Apr 2020 14:12:24 -0700 Subject: [PATCH 05/22] remove unused fn --- .../aggregation/post/FieldAccessPostAggregator.java | 1 + .../org/apache/druid/segment/column/ValueTypes.java | 12 ------------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java index c2db45290cac..2c129583cfa4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -41,6 +41,7 @@ public class FieldAccessPostAggregator implements PostAggregator @Nullable private final String name; private final String fieldName; + @Nullable private final ValueType type; @JsonCreator diff --git a/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java b/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java index 2285eb1b81dc..b9311dcaa76a 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ValueTypes.java @@ -19,8 +19,6 @@ package org.apache.druid.segment.column; -import org.apache.druid.common.guava.GuavaUtils; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DimensionSelector; @@ -72,16 +70,6 @@ public static DimensionSelector makeNumericWrappingDimensionSelector( } } - public static ValueType aggregatorTypeNameToType(String name) - { - ValueType v = GuavaUtils.getEnumIfPresent( - ValueType.class, - StringUtils.toUpperCase(name) - ); - - return v == null ? ValueType.COMPLEX : v; - } - public static ValueType defaultAggregationType() { return ValueType.DOUBLE; From 21067bc05b08b180f435ff523e1a90a89d33a43d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Sun, 19 Apr 2020 14:20:53 -0700 Subject: [PATCH 06/22] adjust --- .../apache/druid/query/aggregation/AggregatorFactory.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index e0b2d7d518fd..a6a99c1c3129 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -238,7 +238,6 @@ public ValueType getFinalizedType() * * If you need a ValueType enum corresponding to this aggregator, use {@link #getTypeName} instead. */ - @Nullable public String getTypeName() { throw new ISE("Complex type name not is not available for %s of type %s", getName(), getType()); @@ -330,8 +329,6 @@ public static AggregatorFactory[] mergeAggregators(List agg } } - return mergedAggregators == null - ? null - : mergedAggregators.values().toArray(new AggregatorFactory[0]); + return mergedAggregators == null ? null : mergedAggregators.values().toArray(new AggregatorFactory[0]); } } From 3aa9958ae3af08598dc76c251b4a656985140652 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 18 Aug 2020 11:14:05 -0700 Subject: [PATCH 07/22] more better --- .../druid/segment/column/ValueType.java | 17 +++++- .../DistinctCountAggregatorFactory.java | 6 ++ .../MomentSketchAggregatorFactory.java | 6 ++ .../movingaverage/AveragerFactoryWrapper.java | 7 ++- .../TDigestSketchAggregatorFactory.java | 7 ++- .../TDigestGenerateSketchSqlAggregator.java | 3 +- .../TDigestSketchQuantileSqlAggregator.java | 3 +- .../hll/sql/HllSketchBaseSqlAggregator.java | 14 +++-- ...blesSketchApproxQuantileSqlAggregator.java | 3 +- .../sql/DoublesSketchObjectSqlAggregator.java | 3 +- .../sql/ThetaSketchBaseSqlAggregator.java | 14 +++-- ...esSketchToHistogramPostAggregatorTest.java | 2 +- ...yOfDoublesSketchAggregatorFactoryTest.java | 1 + .../bloom/BloomFilterAggregatorFactory.java | 6 ++ .../bloom/sql/BloomFilterSqlAggregator.java | 4 +- .../sql/BloomFilterOperatorConversion.java | 2 +- ...BucketsHistogramQuantileSqlAggregator.java | 3 +- .../histogram/sql/QuantileSqlAggregator.java | 2 +- .../sql/BaseVarianceSqlAggregator.java | 8 +-- .../query/aggregation/AggregatorFactory.java | 7 +-- .../aggregation/CountAggregatorFactory.java | 6 ++ .../JavaScriptAggregatorFactory.java | 6 ++ .../NullableNumericAggregatorFactory.java | 7 +++ .../any/DoubleAnyAggregatorFactory.java | 6 ++ .../any/FloatAnyAggregatorFactory.java | 6 ++ .../any/LongAnyAggregatorFactory.java | 6 ++ .../any/StringAnyAggregatorFactory.java | 6 ++ .../RowBasedColumnSelectorFactory.java | 3 +- .../druid/segment/column/RowSignature.java | 4 +- .../segment/incremental/IncrementalIndex.java | 10 ++-- .../query/metadata/SegmentAnalyzerTest.java | 12 ++++ .../TimeseriesQueryQueryToolChestTest.java | 2 +- .../ApproxCountDistinctSqlAggregator.java | 13 +++-- .../EarliestLatestAnySqlAggregator.java | 8 +-- .../aggregation/builtin/MaxSqlAggregator.java | 2 +- .../aggregation/builtin/MinSqlAggregator.java | 2 +- .../builtin/MultiColumnSqlAggregator.java | 2 +- .../aggregation/builtin/SumSqlAggregator.java | 2 +- .../sql/calcite/expression/Expressions.java | 6 +- .../builtin/LikeOperatorConversion.java | 2 +- .../ReductionOperatorConversionHelper.java | 2 +- .../builtin/RegexpLikeOperatorConversion.java | 2 +- .../druid/sql/calcite/planner/Calcites.java | 55 +++++++++++++++---- .../druid/sql/calcite/rel/DruidQuery.java | 6 +- .../druid/sql/calcite/rel/Projection.java | 12 ++-- .../calcite/rel/VirtualColumnRegistry.java | 25 +++++++-- .../sql/calcite/table/RowSignatures.java | 23 +++++--- .../druid/sql/calcite/CalciteQueryTest.java | 4 +- 48 files changed, 266 insertions(+), 92 deletions(-) diff --git a/core/src/main/java/org/apache/druid/segment/column/ValueType.java b/core/src/main/java/org/apache/druid/segment/column/ValueType.java index 219f7dc63d42..7693f0963dfd 100644 --- a/core/src/main/java/org/apache/druid/segment/column/ValueType.java +++ b/core/src/main/java/org/apache/druid/segment/column/ValueType.java @@ -24,13 +24,26 @@ import javax.annotation.Nullable; +/** + * This enumeration defines the Druid type system used to indicate the type of data stored in columns, produced by + * expressions, used to allow query processing engine algorithms to compute results, used to compute query result + * row signatures, and all other type needs. + * + * Currently only the primitive types ({@link #isPrimitive()} is true) and {@link #COMPLEX} can be stored in columns + * and are the only types handled directly by the query engines. Array types can currently be produced by expressions + * and by some post-aggregators, but do not currently have special engine handling, and should be used by implementors + * sparingly until full engine support is in place. + */ public enum ValueType { + // primitive types DOUBLE, FLOAT, LONG, STRING, + // non-primitive types COMPLEX, + // transient array types (also non-primitive) DOUBLE_ARRAY, LONG_ARRAY, STRING_ARRAY; @@ -41,14 +54,14 @@ public boolean isNumeric() return isNumeric(this); } - public boolean isPrimitiveScalar() + public boolean isPrimitive() { return this.equals(ValueType.STRING) || isNumeric(this); } public boolean isComplex() { - return !isPrimitiveScalar(); + return !isPrimitive(); } @Nullable diff --git a/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java b/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java index e3b5d51d35a6..568864a3775a 100644 --- a/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java @@ -211,6 +211,12 @@ public ValueType getType() return ValueType.LONG; } + @Override + public ValueType getFinalizedType() + { + return ValueType.LONG; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java index 8d93f108b803..c78a3bc2dec3 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java @@ -254,6 +254,12 @@ public ValueType getType() return ValueType.COMPLEX; } + @Override + public ValueType getFinalizedType() + { + return ValueType.COMPLEX; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java index c3253e46fef2..a2e89103343b 100644 --- a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java +++ b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java @@ -178,6 +178,12 @@ public ValueType getType() return ValueType.COMPLEX; } + @Override + public ValueType getFinalizedType() + { + return getType(); + } + /** * Not implemented. Throws UnsupportedOperationException. */ @@ -186,5 +192,4 @@ public int getMaxIntermediateSize() { throw new UnsupportedOperationException("Invalid operation for AveragerFactoryWrapper."); } - } diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java index e0e73d881a25..49be6161107d 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java @@ -226,6 +226,12 @@ public ValueType getType() return ValueType.COMPLEX; } + @Override + public ValueType getFinalizedType() + { + return ValueType.COMPLEX; + } + @Override public int getMaxIntermediateSize() { @@ -305,5 +311,4 @@ public String toString() + ", compression=" + compression + "}"; } - } diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java index a7346ffb0ffb..0bbc5488881e 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestGenerateSketchSqlAggregator.java @@ -37,6 +37,7 @@ import org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchUtils; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; import org.apache.druid.sql.calcite.expression.DruidExpression; @@ -139,7 +140,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, input, - SqlTypeName.FLOAT + ValueType.FLOAT ); virtualColumns.add(virtualColumn); aggregatorFactory = new TDigestSketchAggregatorFactory( diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java index 09d8b02fecab..3fd1e3787679 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchQuantileSqlAggregator.java @@ -40,6 +40,7 @@ import org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchUtils; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; import org.apache.druid.sql.calcite.expression.DruidExpression; @@ -158,7 +159,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, input, - SqlTypeName.FLOAT + ValueType.FLOAT ); virtualColumns.add(virtualColumn); aggregatorFactory = new TDigestSketchAggregatorFactory( diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java index 87ea9038de5f..2f08cf0cdecc 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchBaseSqlAggregator.java @@ -21,11 +21,11 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.datasketches.hll.HllSketchAggregatorFactory; @@ -129,10 +129,14 @@ public Aggregation toDruidAggregation( ROUND ); } else { - final SqlTypeName sqlTypeName = columnRexNode.getType().getSqlTypeName(); - final ValueType inputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + final RelDataType dataType = columnRexNode.getType(); + final ValueType inputType = Calcites.getValueTypeForRelDataType(dataType); if (inputType == null) { - throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, aggregatorName); + throw new ISE( + "Cannot translate sqlTypeName[%s] to Druid type for field[%s]", + dataType.getSqlTypeName(), + aggregatorName + ); } final DimensionSpec dimensionSpec; @@ -143,7 +147,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, columnArg, - sqlTypeName + dataType ); dimensionSpec = new DefaultDimensionSpec(virtualColumn.getOutputName(), null, inputType); virtualColumns.add(virtualColumn); diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchApproxQuantileSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchApproxQuantileSqlAggregator.java index 47aade4683cc..62f97fd12ed1 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchApproxQuantileSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchApproxQuantileSqlAggregator.java @@ -39,6 +39,7 @@ import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; @@ -182,7 +183,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, input, - SqlTypeName.FLOAT + ValueType.FLOAT ); virtualColumns.add(virtualColumn); aggregatorFactory = new DoublesSketchAggregatorFactory( diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchObjectSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchObjectSqlAggregator.java index 70b5eb16cb33..ba0493c4c08d 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchObjectSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchObjectSqlAggregator.java @@ -37,6 +37,7 @@ import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchAggregatorFactory; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; import org.apache.druid.sql.calcite.expression.DruidExpression; @@ -120,7 +121,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, input, - SqlTypeName.FLOAT + ValueType.FLOAT ); virtualColumns.add(virtualColumn); aggregatorFactory = new DoublesSketchAggregatorFactory( diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchBaseSqlAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchBaseSqlAggregator.java index ac277ded842d..ed2aafc57825 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchBaseSqlAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchBaseSqlAggregator.java @@ -21,11 +21,11 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.datasketches.theta.SketchAggregatorFactory; @@ -109,10 +109,14 @@ public Aggregation toDruidAggregation( null ); } else { - final SqlTypeName sqlTypeName = columnRexNode.getType().getSqlTypeName(); - final ValueType inputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + final RelDataType dataType = columnRexNode.getType(); + final ValueType inputType = Calcites.getValueTypeForRelDataType(dataType); if (inputType == null) { - throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, aggregatorName); + throw new ISE( + "Cannot translate sqlTypeName[%s] to Druid type for field[%s]", + dataType.getSqlTypeName(), + aggregatorName + ); } final DimensionSpec dimensionSpec; @@ -123,7 +127,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, columnArg, - sqlTypeName + dataType ); dimensionSpec = new DefaultDimensionSpec(virtualColumn.getOutputName(), null, inputType); virtualColumns.add(virtualColumn); diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java index 50f9c1c54551..3163bfc776b3 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java @@ -203,7 +203,7 @@ public void testResultArraySignature() "a", new FieldAccessPostAggregator("field", "sketch"), new double[] {3.5}, - 24 + null ) ) .build(); diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java index 1a6e2c1ded78..d41ef041de5c 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactoryTest.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.datasketches.tuple; +import com.google.common.collect.ImmutableList; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.datasketches.tuple.ArrayOfDoublesSketch; import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch; diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java index 27cee40a57e2..519668c6a45f 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java @@ -193,6 +193,12 @@ public ValueType getType() return ValueType.COMPLEX; } + @Override + public ValueType getFinalizedType() + { + return ValueType.COMPLEX; + } + @Override public int getMaxIntermediateSize() { diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java index f1fca86b09f3..ff7dbd421b72 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregator.java @@ -152,7 +152,7 @@ public Aggregation toDruidAggregation( // No existing match found. Create a new one. final List virtualColumns = new ArrayList<>(); - ValueType valueType = Calcites.getValueTypeForSqlTypeName(inputOperand.getType().getSqlTypeName()); + ValueType valueType = Calcites.getValueTypeForRelDataType(inputOperand.getType()); final DimensionSpec spec; if (input.isDirectColumnAccess()) { spec = new DefaultDimensionSpec( @@ -171,7 +171,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, input, - inputOperand.getType().getSqlTypeName() + inputOperand.getType() ); virtualColumns.add(virtualColumn); spec = new DefaultDimensionSpec( diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java index 8041f02c90a6..edd6430aea22 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/filter/sql/BloomFilterOperatorConversion.java @@ -107,7 +107,7 @@ public DimFilter toDruidFilter( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, - operands.get(0).getType().getSqlTypeName() + operands.get(0).getType() ); if (virtualColumn == null) { return null; diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregator.java index cd7e58b15c62..283742306bcb 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregator.java @@ -39,6 +39,7 @@ import org.apache.druid.query.aggregation.histogram.QuantilePostAggregator; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.SqlAggregator; @@ -238,7 +239,7 @@ public Aggregation toDruidAggregation( VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, input, - SqlTypeName.FLOAT + ValueType.FLOAT ); virtualColumns.add(virtualColumn); aggregatorFactory = new FixedBucketsHistogramAggregatorFactory( diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java index 44dd9a67c36e..abbd2aca2fc5 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java @@ -198,7 +198,7 @@ public Aggregation toDruidAggregation( } } else { final VirtualColumn virtualColumn = - virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.FLOAT); + virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, ValueType.FLOAT); virtualColumns.add(virtualColumn); aggregatorFactory = new ApproximateHistogramAggregatorFactory( histogramName, diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/sql/BaseVarianceSqlAggregator.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/sql/BaseVarianceSqlAggregator.java index 36d1a189ccb7..43e6c10898ba 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/sql/BaseVarianceSqlAggregator.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/sql/BaseVarianceSqlAggregator.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableList; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -81,8 +81,8 @@ public Aggregation toDruidAggregation( } final AggregatorFactory aggregatorFactory; - final SqlTypeName sqlTypeName = inputOperand.getType().getSqlTypeName(); - final ValueType inputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + final RelDataType dataType = inputOperand.getType(); + final ValueType inputType = Calcites.getValueTypeForRelDataType(dataType); final List virtualColumns = new ArrayList<>(); final DimensionSpec dimensionSpec; final String aggName = StringUtils.format("%s:agg", name); @@ -95,7 +95,7 @@ public Aggregation toDruidAggregation( dimensionSpec = input.getSimpleExtraction().toDimensionSpec(null, inputType); } else { VirtualColumn virtualColumn = - virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, sqlTypeName); + virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, dataType); dimensionSpec = new DefaultDimensionSpec(virtualColumn.getOutputName(), null, inputType); virtualColumns.add(virtualColumn); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index ab07566addcf..0331d316865c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -225,10 +225,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * and {@link #combine}. * @return */ - public ValueType getFinalizedType() - { - return getType(); - } + public abstract ValueType getFinalizedType(); /** * Get the complex type name of the intermediate type for this aggregator. @@ -238,6 +235,8 @@ public ValueType getFinalizedType() * {@link org.apache.druid.segment.serde.ComplexMetrics#registerSerde} using this type name. * * If you need a ValueType enum corresponding to this aggregator, use {@link #getTypeName} instead. + * + * @throws IllegalStateException if getType() != ValueType.COMPLEX */ public String getTypeName() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java index dc0cbb2b617e..95002d542150 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/CountAggregatorFactory.java @@ -141,6 +141,12 @@ public ValueType getType() return ValueType.LONG; } + @Override + public ValueType getFinalizedType() + { + return ValueType.LONG; + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java index 74027d7c2661..ef3c151aa16b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -273,6 +273,12 @@ public ValueType getType() return ValueType.FLOAT; } + @Override + public ValueType getFinalizedType() + { + return ValueType.FLOAT; + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/NullableNumericAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/NullableNumericAggregatorFactory.java index 615f0b57db1a..551717d2e257 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/NullableNumericAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/NullableNumericAggregatorFactory.java @@ -26,6 +26,7 @@ import org.apache.druid.segment.BaseNullableColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -145,4 +146,10 @@ protected VectorAggregator factorizeVector( throw new UnsupportedOperationException("canVectorize returned true but 'factorizeVector' is not implemented"); } } + + @Override + public ValueType getFinalizedType() + { + return getType(); + } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java index c8c31a3fc866..ac962f3bde03 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/DoubleAnyAggregatorFactory.java @@ -193,6 +193,12 @@ public ValueType getType() return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } + @Override + public ValueType getFinalizedType() + { + return getType(); + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java index 727eea1b613b..d7a912d8abdb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/FloatAnyAggregatorFactory.java @@ -191,6 +191,12 @@ public ValueType getType() return ValueType.FLOAT; } + @Override + public ValueType getFinalizedType() + { + return ValueType.FLOAT; + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java index ceb7d83568a6..10bf526a7dba 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/LongAnyAggregatorFactory.java @@ -189,6 +189,12 @@ public ValueType getType() return ValueType.LONG; } + @Override + public ValueType getFinalizedType() + { + return ValueType.LONG; + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java index 7be9fb7cafc3..97296ec18d5b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java @@ -153,6 +153,12 @@ public ValueType getType() return ValueType.STRING; } + @Override + public ValueType getFinalizedType() + { + return ValueType.STRING; + } + @Override public int getMaxIntermediateSize() { diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java index 77e8978e2bee..b608cbcb6735 100644 --- a/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java @@ -102,8 +102,7 @@ static ColumnCapabilities getColumnCapabilities( // Do _not_ set isDictionaryEncoded or hasBitmapIndexes, because Row-based columns do not have those things. // Do not set hasMultipleValues, because even though we might return multiple values, setting it affirmatively - // causes expression selectors to always treat us as arrays. If we might have multiple values (i.e. if our type - // is nonnumeric), set isComplete false to compensate. + // causes expression selectors to always treat us as arrays, so leave as unknown. if (valueType != null) { if (valueType.isNumeric()) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index d2dde2cdc02f..d1853bd0c4c8 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -236,13 +236,13 @@ public Builder addAggregators(final List aggregators) { for (final AggregatorFactory aggregator : aggregators) { final ValueType type = aggregator.getType(); - + if (type.equals(aggregator.getFinalizedType())) { add(aggregator.getName(), type); } else { // Use null if the type depends on whether or not the aggregator is finalized, since // we don't know if it will be finalized or not. So null (i.e. unknown) is the proper - // thing to do. + // thing to do (currently). add(aggregator.getName(), null); } } diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java index c69444379c5c..487f0115f94f 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java @@ -1119,7 +1119,10 @@ public MetricDesc(int index, AggregatorFactory factory) ValueType valueType = factory.getType(); - if (valueType.isComplex()) { + if (valueType.isPrimitive()) { + capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); + this.type = valueType.toString(); + } else { capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.COMPLEX) .setHasNulls(ColumnCapabilities.Capable.TRUE); String complexTypeName = factory.getTypeName(); @@ -1127,11 +1130,8 @@ public MetricDesc(int index, AggregatorFactory factory) if (serde != null) { this.type = serde.getTypeName(); } else { - throw new ISE("Don't know how to handle type[%s]", complexTypeName); + throw new ISE("Don't know how to handle complex type[%s] of type[%s]", complexTypeName, valueType); } - } else { - capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.FLOAT); - this.type = valueType.toString(); } } diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java index 58e732382cbe..32a3958a64d8 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java @@ -413,6 +413,12 @@ public List requiredFields() return Collections.singletonList(fieldName); } + @Override + public ValueType getType() + { + return ValueType.COMPLEX; + } + @Override public String getTypeName() { @@ -430,6 +436,12 @@ public byte[] getCacheKey() { return new byte[0]; } + + @Override + public ValueType getFinalizedType() + { + return getType(); + } } } diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java index 70d0ac585eb6..483d907b7dc4 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java @@ -410,7 +410,7 @@ public void testResultArraySignatureWithTimestampResultField() .add("rows", ValueType.LONG) .add("index", ValueType.DOUBLE) .add("uniques", null) - .add("const", null) + .add("const", ValueType.LONG) .build(), TOOL_CHEST.resultArraySignature(query) ); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ApproxCountDistinctSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ApproxCountDistinctSqlAggregator.java index 2c15bf214e27..73bcecaab271 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ApproxCountDistinctSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/ApproxCountDistinctSqlAggregator.java @@ -23,6 +23,7 @@ import com.google.common.collect.Iterables; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlAggFunction; @@ -101,10 +102,14 @@ public Aggregation toDruidAggregation( && rowSignature.getColumnType(arg.getDirectColumn()).orElse(null) == ValueType.COMPLEX) { aggregatorFactory = new HyperUniquesAggregatorFactory(aggregatorName, arg.getDirectColumn(), false, true); } else { - final SqlTypeName sqlTypeName = rexNode.getType().getSqlTypeName(); - final ValueType inputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + final RelDataType dataType = rexNode.getType(); + final ValueType inputType = Calcites.getValueTypeForRelDataType(dataType); if (inputType == null) { - throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, aggregatorName); + throw new ISE( + "Cannot translate sqlTypeName[%s] to Druid type for field[%s]", + dataType.getSqlTypeName(), + aggregatorName + ); } final DimensionSpec dimensionSpec; @@ -113,7 +118,7 @@ public Aggregation toDruidAggregation( dimensionSpec = arg.getSimpleExtraction().toDimensionSpec(null, inputType); } else { VirtualColumn virtualColumn = - virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, arg, sqlTypeName); + virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, arg, dataType); dimensionSpec = new DefaultDimensionSpec(virtualColumn.getOutputName(), null, inputType); myvirtualColumns.add(virtualColumn); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java index 063ea1b70568..800ee7d0c26c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java @@ -21,6 +21,7 @@ import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; @@ -30,7 +31,6 @@ import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.Optionality; import org.apache.druid.java.util.common.ISE; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -185,16 +185,16 @@ public Aggregation toDruidAggregation( if (args.get(0).isDirectColumnAccess()) { fieldName = args.get(0).getDirectColumn(); } else { - final SqlTypeName sqlTypeName = rexNodes.get(0).getType().getSqlTypeName(); + final RelDataType dataType = rexNodes.get(0).getType(); final VirtualColumn virtualColumn = - virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, args.get(0), sqlTypeName); + virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, args.get(0), dataType); fieldName = virtualColumn.getOutputName(); } // Second arg must be a literal, if it exists (the type signature below requires it). final int maxBytes = rexNodes.size() > 1 ? RexLiteral.intValue(rexNodes.get(1)) : -1; - final ValueType outputType = Calcites.getValueTypeForSqlTypeName(aggregateCall.getType().getSqlTypeName()); + final ValueType outputType = Calcites.getValueTypeForRelDataType(aggregateCall.getType()); if (outputType == null) { throw new ISE( "Cannot translate output sqlTypeName[%s] to Druid type for aggregator[%s]", diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MaxSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MaxSqlAggregator.java index c55533002ffb..302560c9bf47 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MaxSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MaxSqlAggregator.java @@ -49,7 +49,7 @@ Aggregation getAggregation( final String expression ) { - final ValueType valueType = Calcites.getValueTypeForSqlTypeName(aggregateCall.getType().getSqlTypeName()); + final ValueType valueType = Calcites.getValueTypeForRelDataType(aggregateCall.getType()); return Aggregation.create(createMaxAggregatorFactory(valueType, name, fieldName, expression, macroTable)); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MinSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MinSqlAggregator.java index d1d2d73f6d91..9e41ce5474b9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MinSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MinSqlAggregator.java @@ -49,7 +49,7 @@ Aggregation getAggregation( final String expression ) { - final ValueType valueType = Calcites.getValueTypeForSqlTypeName(aggregateCall.getType().getSqlTypeName()); + final ValueType valueType = Calcites.getValueTypeForRelDataType(aggregateCall.getType()); return Aggregation.create(createMinAggregatorFactory(valueType, name, fieldName, expression, macroTable)); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MultiColumnSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MultiColumnSqlAggregator.java index 4c7efb7c0b01..78a3e160f1ce 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MultiColumnSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/MultiColumnSqlAggregator.java @@ -122,7 +122,7 @@ private Aggregation getAggregation( List fieldInfoList ) { - final ValueType valueType = Calcites.getValueTypeForSqlTypeName(aggregateCall.getType().getSqlTypeName()); + final ValueType valueType = Calcites.getValueTypeForRelDataType(aggregateCall.getType()); List aggregatorFactories = new ArrayList<>(); List postAggregators = new ArrayList<>(); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/SumSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/SumSqlAggregator.java index c96d74e59223..b9a5af57d003 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/SumSqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/SumSqlAggregator.java @@ -49,7 +49,7 @@ Aggregation getAggregation( final String expression ) { - final ValueType valueType = Calcites.getValueTypeForSqlTypeName(aggregateCall.getType().getSqlTypeName()); + final ValueType valueType = Calcites.getValueTypeForRelDataType(aggregateCall.getType()); return Aggregation.create(createSumAggregatorFactory(valueType, name, fieldName, expression, macroTable)); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java index b0c7c59421e7..dc657f9688fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java @@ -470,7 +470,7 @@ private static DimFilter toSimpleLeafFilter( final VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, - operand.getType().getSqlTypeName() + operand.getType() ); equalFilter = new SelectorDimFilter( @@ -559,7 +559,7 @@ private static DimFilter toSimpleLeafFilter( VirtualColumn virtualLhs = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, lhsExpression, - lhs.getType().getSqlTypeName() + lhs.getType() ); column = virtualLhs.getOutputName(); @@ -607,7 +607,7 @@ private static DimFilter toSimpleLeafFilter( } // Numeric lhs needs a numeric comparison. - final StringComparator comparator = Calcites.getStringComparatorForSqlTypeName(lhs.getType().getSqlTypeName()); + final StringComparator comparator = Calcites.getStringComparatorForRelDataType(lhs.getType()); final BoundRefKey boundRefKey = new BoundRefKey(column, extractionFn, comparator); final DimFilter filter; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/LikeOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/LikeOperatorConversion.java index eb7350840eee..88fb4b013328 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/LikeOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/LikeOperatorConversion.java @@ -82,7 +82,7 @@ public DimFilter toDruidFilter( VirtualColumn v = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, - operands.get(0).getType().getSqlTypeName() + operands.get(0).getType() ); return new LikeDimFilter( diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ReductionOperatorConversionHelper.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ReductionOperatorConversionHelper.java index 595955e1c0d5..f76d8352bbb5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ReductionOperatorConversionHelper.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ReductionOperatorConversionHelper.java @@ -55,7 +55,7 @@ private ReductionOperatorConversionHelper() for (int i = 0; i < n; i++) { RelDataType type = opBinding.getOperandType(i); SqlTypeName sqlTypeName = type.getSqlTypeName(); - ValueType valueType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + ValueType valueType = Calcites.getValueTypeForRelDataType(type); // Return types are listed in order of preference: if (valueType == ValueType.STRING) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/RegexpLikeOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/RegexpLikeOperatorConversion.java index ea699abe0215..4688456743d1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/RegexpLikeOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/RegexpLikeOperatorConversion.java @@ -105,7 +105,7 @@ public DimFilter toDruidFilter( VirtualColumn v = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, - operands.get(0).getType().getSqlTypeName() + operands.get(0).getType() ); return new RegexDimFilter(v.getOutputName(), pattern, null, null); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java index 9ba5214446ff..05a7f4d8da97 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/Calcites.java @@ -127,32 +127,48 @@ public static String escapeStringLiteral(final String s) } @Nullable - public static ValueType getValueTypeForSqlTypeName(SqlTypeName sqlTypeName) + public static ValueType getValueTypeForRelDataType(final RelDataType type) { + final SqlTypeName sqlTypeName = type.getSqlTypeName(); if (SqlTypeName.FLOAT == sqlTypeName) { return ValueType.FLOAT; - } else if (SqlTypeName.FRACTIONAL_TYPES.contains(sqlTypeName)) { + } else if (isDoubleType(sqlTypeName)) { return ValueType.DOUBLE; - } else if (SqlTypeName.TIMESTAMP == sqlTypeName - || SqlTypeName.DATE == sqlTypeName - || SqlTypeName.BOOLEAN == sqlTypeName - || SqlTypeName.INT_TYPES.contains(sqlTypeName)) { + } else if (isLongType(sqlTypeName)) { return ValueType.LONG; } else if (SqlTypeName.CHAR_TYPES.contains(sqlTypeName)) { return ValueType.STRING; } else if (SqlTypeName.OTHER == sqlTypeName) { return ValueType.COMPLEX; } else if (sqlTypeName == SqlTypeName.ARRAY) { - // until we have array ValueType, this will let us have array constants and use them at least - return ValueType.STRING; + SqlTypeName componentType = type.getComponentType().getSqlTypeName(); + if (isDoubleType(componentType)) { + return ValueType.DOUBLE_ARRAY; + } + if (isLongType(componentType)) { + return ValueType.LONG_ARRAY; + } + return ValueType.STRING_ARRAY; } else { return null; } } - public static StringComparator getStringComparatorForSqlTypeName(SqlTypeName sqlTypeName) + public static boolean isDoubleType(SqlTypeName sqlTypeName) + { + return SqlTypeName.FRACTIONAL_TYPES.contains(sqlTypeName) || SqlTypeName.APPROX_TYPES.contains(sqlTypeName); + } + public static boolean isLongType(SqlTypeName sqlTypeName) + { + return SqlTypeName.TIMESTAMP == sqlTypeName || + SqlTypeName.DATE == sqlTypeName || + SqlTypeName.BOOLEAN == sqlTypeName || + SqlTypeName.INT_TYPES.contains(sqlTypeName); + } + + public static StringComparator getStringComparatorForRelDataType(RelDataType dataType) { - final ValueType valueType = getValueTypeForSqlTypeName(sqlTypeName); + final ValueType valueType = getValueTypeForRelDataType(dataType); return getStringComparatorForValueType(valueType); } @@ -207,6 +223,25 @@ public static RelDataType createSqlTypeWithNullability( return typeFactory.createTypeWithNullability(dataType, nullable); } + /** + * Like RelDataTypeFactory.createSqlTypeWithNullability, but creates types that align best with how Druid + * represents them. + */ + public static RelDataType createSqlArrayTypeWithNullability( + final RelDataTypeFactory typeFactory, + final SqlTypeName elementTypeName, + final boolean nullable + ) + { + + final RelDataType dataType = typeFactory.createArrayType( + createSqlTypeWithNullability(typeFactory, elementTypeName, nullable), + -1 + ); + + return dataType; + } + /** * Calcite expects "TIMESTAMP" types to be an instant that has the expected local time fields if printed as UTC. * diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 45c29d1787a1..8fc59f3e3747 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -391,8 +391,8 @@ private static List computeDimensions( throw new CannotBuildQueryException(aggregate, rexNode); } - final SqlTypeName sqlTypeName = rexNode.getType().getSqlTypeName(); - final ValueType outputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + final RelDataType dataType = rexNode.getType(); + final ValueType outputType = Calcites.getValueTypeForRelDataType(dataType); if (outputType == null || outputType == ValueType.COMPLEX) { // Can't group on unknown or COMPLEX types. throw new CannotBuildQueryException(aggregate, rexNode); @@ -406,7 +406,7 @@ private static List computeDimensions( virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, druidExpression, - sqlTypeName + dataType ); dimensions.add(DimensionExpression.ofVirtualColumn( virtualColumn.getOutputName(), diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java index be52552df9b4..d353a4ae116c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java @@ -22,9 +22,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.math.expr.ExprType; @@ -260,17 +260,17 @@ public static Projection preAggregation( for (int i = 0; i < expressions.size(); i++) { final DruidExpression expression = expressions.get(i); - final SqlTypeName sqlTypeName = project.getRowType().getFieldList().get(i).getType().getSqlTypeName(); + final RelDataType dataType = project.getRowType().getFieldList().get(i).getType(); if (expression.isDirectColumnAccess() && inputRowSignature.getColumnType(expression.getDirectColumn()).orElse(null) - == Calcites.getValueTypeForSqlTypeName(sqlTypeName)) { + == Calcites.getValueTypeForRelDataType(dataType)) { // Refer to column directly when it's a direct access with matching type. rowOrder.add(expression.getDirectColumn()); } else { final VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( plannerContext, expression, - project.getChildExps().get(i).getType().getSqlTypeName() + project.getChildExps().get(i).getType() ); virtualColumns.add(virtualColumn); rowOrder.add(virtualColumn.getOutputName()); @@ -318,7 +318,7 @@ private static boolean postAggregatorDirectColumnIsOk( // Check if a cast is necessary. final ExprType toExprType = Expressions.exprTypeForValueType(columnValueType); final ExprType fromExprType = Expressions.exprTypeForValueType( - Calcites.getValueTypeForSqlTypeName(rexNode.getType().getSqlTypeName()) + Calcites.getValueTypeForRelDataType(rexNode.getType()) ); return toExprType.equals(fromExprType); @@ -351,7 +351,7 @@ private static boolean postAggregatorComplexDirectColumnIsOk( () -> new ISE("Encountered null type for column[%s]", expression.getDirectColumn()) ); - final ValueType fromValueType = Calcites.getValueTypeForSqlTypeName(rexNode.getType().getSqlTypeName()); + final ValueType fromValueType = Calcites.getValueTypeForRelDataType(rexNode.getType()); return toValueType == ValueType.COMPLEX && fromValueType == ValueType.COMPLEX; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java index 699fb74d62b4..d6ff84003037 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java @@ -19,9 +19,10 @@ package org.apache.druid.sql.calcite.rel; -import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.rel.type.RelDataType; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.PlannerContext; @@ -74,19 +75,19 @@ public boolean isVirtualColumnDefined(String virtualColumnName) } /** - * Get existing or create new {@link VirtualColumn} for a given {@link DruidExpression}. + * Get existing or create new {@link VirtualColumn} for a given {@link DruidExpression} and {@link ValueType}. */ public VirtualColumn getOrCreateVirtualColumnForExpression( PlannerContext plannerContext, DruidExpression expression, - SqlTypeName typeName + ValueType valueType ) { if (!virtualColumnsByExpression.containsKey(expression.getExpression())) { final String virtualColumnName = virtualColumnPrefix + virtualColumnCounter++; final VirtualColumn virtualColumn = expression.toVirtualColumn( virtualColumnName, - Calcites.getValueTypeForSqlTypeName(typeName), + valueType, plannerContext.getExprMacroTable() ); virtualColumnsByExpression.put( @@ -102,6 +103,22 @@ public VirtualColumn getOrCreateVirtualColumnForExpression( return virtualColumnsByExpression.get(expression.getExpression()); } + /** + * Get existing or create new {@link VirtualColumn} for a given {@link DruidExpression} and {@link RelDataType} + */ + public VirtualColumn getOrCreateVirtualColumnForExpression( + PlannerContext plannerContext, + DruidExpression expression, + RelDataType dataType + ) + { + return getOrCreateVirtualColumnForExpression( + plannerContext, + expression, + Calcites.getValueTypeForRelDataType(dataType) + ); + } + /** * Get existing virtual column by column name */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java b/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java index 97d6ed9b3d5e..91d19747b271 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/table/RowSignatures.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.IAE; @@ -57,13 +56,14 @@ public static RowSignature fromRelDataType(final List rowOrder, final Re final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); for (int i = 0; i < rowOrder.size(); i++) { - final RelDataTypeField field = rowType.getFieldList().get(i); - final SqlTypeName sqlTypeName = field.getType().getSqlTypeName(); - final ValueType valueType; - - valueType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); + final RelDataType dataType = rowType.getFieldList().get(i).getType(); + final ValueType valueType = Calcites.getValueTypeForRelDataType(dataType); if (valueType == null) { - throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, rowOrder.get(i)); + throw new ISE( + "Cannot translate sqlTypeName[%s] to Druid type for field[%s]", + dataType.getSqlTypeName(), + rowOrder.get(i) + ); } rowSignatureBuilder.add(rowOrder.get(i), valueType); @@ -122,6 +122,15 @@ public static RelDataType toRelDataType(final RowSignature rowSignature, final R case DOUBLE: type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.DOUBLE, nullNumeric); break; + case STRING_ARRAY: + type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.VARCHAR, true); + break; + case LONG_ARRAY: + type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.BIGINT, nullNumeric); + break; + case DOUBLE_ARRAY: + type = Calcites.createSqlArrayTypeWithNullability(typeFactory, SqlTypeName.DOUBLE, nullNumeric); + break; case COMPLEX: // Loses information about exactly what kind of complex column this is. type = Calcites.createSqlTypeWithNullability(typeFactory, SqlTypeName.OTHER, true); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 47d7777674ae..d27f8590593f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -14009,7 +14009,7 @@ public void testSelectConstantArrayExpressionFromTable() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "array(1,2)", ValueType.STRING)) + .virtualColumns(expressionVirtualColumn("v0", "array(1,2)", ValueType.LONG_ARRAY)) .columns("dim1", "v0") .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) .limit(1) @@ -14031,7 +14031,7 @@ public void testSelectNonConstantArrayExpressionFromTable() throws Exception newScanQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) .intervals(querySegmentSpec(Filtration.eternity())) - .virtualColumns(expressionVirtualColumn("v0", "array(concat(\"dim1\",'word'),'up')", ValueType.STRING)) + .virtualColumns(expressionVirtualColumn("v0", "array(concat(\"dim1\",'word'),'up')", ValueType.STRING_ARRAY)) .columns("dim1", "v0") .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) .limit(5) From bbf60e2862c9cd85f13bccb41e610324e4430c2c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 18 Aug 2020 12:31:50 -0700 Subject: [PATCH 08/22] rename getTypeName to getComplexTypeName --- .../distinctcount/DistinctCountAggregatorFactory.java | 2 +- .../aggregator/MomentSketchAggregatorFactory.java | 2 +- .../druid/query/movingaverage/AveragerFactoryWrapper.java | 2 +- .../tdigestsketch/TDigestSketchAggregatorFactory.java | 2 +- .../datasketches/hll/HllSketchBuildAggregatorFactory.java | 2 +- .../datasketches/hll/HllSketchMergeAggregatorFactory.java | 2 +- .../quantiles/DoublesSketchAggregatorFactory.java | 2 +- .../datasketches/theta/SketchMergeAggregatorFactory.java | 2 +- .../tuple/ArrayOfDoublesSketchAggregatorFactory.java | 2 +- .../datasketches/hll/HllSketchAggregatorFactoryTest.java | 2 +- .../query/aggregation/bloom/BloomFilterAggregatorFactory.java | 2 +- .../histogram/ApproximateHistogramAggregatorFactory.java | 2 +- .../histogram/FixedBucketsHistogramAggregatorFactory.java | 2 +- .../query/aggregation/variance/VarianceAggregatorFactory.java | 2 +- .../src/main/java/org/apache/druid/indexer/InputRowSerde.java | 4 ++-- .../org/apache/druid/query/aggregation/AggregatorFactory.java | 4 ++-- .../druid/query/aggregation/FilteredAggregatorFactory.java | 4 ++-- .../druid/query/aggregation/HistogramAggregatorFactory.java | 2 +- .../druid/query/aggregation/SuppressedAggregatorFactory.java | 4 ++-- .../aggregation/cardinality/CardinalityAggregatorFactory.java | 2 +- .../query/aggregation/first/DoubleFirstAggregatorFactory.java | 2 +- .../query/aggregation/first/StringFirstAggregatorFactory.java | 2 +- .../hyperloglog/HyperUniquesAggregatorFactory.java | 2 +- .../query/aggregation/last/DoubleLastAggregatorFactory.java | 2 +- .../query/aggregation/last/StringLastAggregatorFactory.java | 2 +- .../query/aggregation/mean/DoubleMeanAggregatorFactory.java | 2 +- .../apache/druid/segment/incremental/IncrementalIndex.java | 4 ++-- .../org/apache/druid/query/metadata/SegmentAnalyzerTest.java | 2 +- 28 files changed, 33 insertions(+), 33 deletions(-) diff --git a/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java b/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java index 568864a3775a..bf6676569b56 100644 --- a/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/org/apache/druid/query/aggregation/distinctcount/DistinctCountAggregatorFactory.java @@ -195,7 +195,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { return "distinctCount"; } diff --git a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java index c78a3bc2dec3..e769302c6939 100644 --- a/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java +++ b/extensions-contrib/momentsketch/src/main/java/org/apache/druid/query/aggregation/momentsketch/aggregator/MomentSketchAggregatorFactory.java @@ -240,7 +240,7 @@ public List requiredFields() } @Override - public String getTypeName() + public String getComplexTypeName() { return TYPE_NAME; } diff --git a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java index a2e89103343b..8467bb97ed00 100644 --- a/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java +++ b/extensions-contrib/moving-average-query/src/main/java/org/apache/druid/query/movingaverage/AveragerFactoryWrapper.java @@ -167,7 +167,7 @@ public byte[] getCacheKey() * return type to be treated as unknown. */ @Override - public String getTypeName() + public String getComplexTypeName() { return ValueType.COMPLEX.name(); } diff --git a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java index 49be6161107d..c63c193f7876 100644 --- a/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java +++ b/extensions-contrib/tdigestsketch/src/main/java/org/apache/druid/query/aggregation/tdigestsketch/TDigestSketchAggregatorFactory.java @@ -212,7 +212,7 @@ public List requiredFields() } @Override - public String getTypeName() + public String getComplexTypeName() { return TYPE_NAME; } diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java index f8e0d6c8b434..8abc305304ed 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildAggregatorFactory.java @@ -51,7 +51,7 @@ public HllSketchBuildAggregatorFactory( } @Override - public String getTypeName() + public String getComplexTypeName() { return HllSketchModule.BUILD_TYPE_NAME; } diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java index ae8cb261e24c..74afea31bfea 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchMergeAggregatorFactory.java @@ -72,7 +72,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre } @Override - public String getTypeName() + public String getComplexTypeName() { return HllSketchModule.MERGE_TYPE_NAME; } diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 6227385b8764..2ea6a2a45fb9 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -248,7 +248,7 @@ public Object finalizeComputation(@Nullable final Object object) } @Override - public String getTypeName() + public String getComplexTypeName() { return DoublesSketchModule.DOUBLES_SKETCH; } diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java index 692069449e1e..87aa733a4738 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchMergeAggregatorFactory.java @@ -141,7 +141,7 @@ public Object finalizeComputation(@Nullable Object object) } @Override - public String getTypeName() + public String getComplexTypeName() { if (isInputThetaSketch) { return SketchModule.THETA_SKETCH_MERGE_AGG; diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java index dc2549598959..cfc8c2e5c00f 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregatorFactory.java @@ -294,7 +294,7 @@ public Object finalizeComputation(@Nullable final Object object) } @Override - public String getTypeName() + public String getComplexTypeName() { if (metricColumns == null) { return ArrayOfDoublesSketchModule.ARRAY_OF_DOUBLES_SKETCH_MERGE_AGG; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java index 5ad3f2967177..6af99b3298f3 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchAggregatorFactoryTest.java @@ -351,7 +351,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) } @Override - public String getTypeName() + public String getComplexTypeName() { return DUMMY_TYPE_NAME; } diff --git a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java index 519668c6a45f..c596141f714a 100644 --- a/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java +++ b/extensions-core/druid-bloom-filter/src/main/java/org/apache/druid/query/aggregation/bloom/BloomFilterAggregatorFactory.java @@ -179,7 +179,7 @@ public List requiredFields() } @Override - public String getTypeName() + public String getComplexTypeName() { return BloomFilterSerializersModule.BLOOM_FILTER_TYPE_NAME; } diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index b9bcced0aab1..c961f504ab33 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -300,7 +300,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { return "approximateHistogram"; } diff --git a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java index d4f8b8d10776..7cd8de80ca64 100644 --- a/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java +++ b/extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/FixedBucketsHistogramAggregatorFactory.java @@ -247,7 +247,7 @@ public List requiredFields() } @Override - public String getTypeName() + public String getComplexTypeName() { return FixedBucketsHistogramAggregator.TYPE_NAME; } diff --git a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java index 0a2f1f19d1bd..4a415061b1d5 100644 --- a/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java +++ b/extensions-core/stats/src/main/java/org/apache/druid/query/aggregation/variance/VarianceAggregatorFactory.java @@ -87,7 +87,7 @@ public VarianceAggregatorFactory(String name, String fieldName) } @Override - public String getTypeName() + public String getComplexTypeName() { return VARIANCE_TYPE_NAME; } diff --git a/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java index 13624242da6d..b3d861061951 100644 --- a/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/org/apache/druid/indexer/InputRowSerde.java @@ -354,7 +354,7 @@ public static SerializeResult toBytes( out.writeDouble(agg.getDouble()); } else if (ValueType.COMPLEX.equals(type)) { Object val = agg.get(); - ComplexMetricSerde serde = getComplexMetricSerde(aggFactory.getTypeName()); + ComplexMetricSerde serde = getComplexMetricSerde(aggFactory.getComplexTypeName()); writeBytes(serde.toBytes(val), out); } else { throw new IAE("Unable to serialize type[%s]", type); @@ -485,7 +485,7 @@ public static InputRow fromBytes( } else if (ValueType.DOUBLE.equals(type)) { event.put(metric, in.readDouble()); } else { - ComplexMetricSerde serde = getComplexMetricSerde(agg.getTypeName()); + ComplexMetricSerde serde = getComplexMetricSerde(agg.getComplexTypeName()); byte[] value = readBytes(in); event.put(metric, serde.fromBytes(value, 0, value.length)); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index 0331d316865c..3d5d2e4a81ff 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -234,11 +234,11 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * must be a corresponding {@link org.apache.druid.segment.serde.ComplexMetricSerde} which was registered with * {@link org.apache.druid.segment.serde.ComplexMetrics#registerSerde} using this type name. * - * If you need a ValueType enum corresponding to this aggregator, use {@link #getTypeName} instead. + * If you need a ValueType enum corresponding to this aggregator, use {@link #getType} instead. * * @throws IllegalStateException if getType() != ValueType.COMPLEX */ - public String getTypeName() + public String getComplexTypeName() { throw new ISE("Complex type name not is not available for %s of type %s", getName(), getType()); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java index a4f86a95f6c5..afdde5b650d4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FilteredAggregatorFactory.java @@ -182,9 +182,9 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { - return delegate.getTypeName(); + return delegate.getComplexTypeName(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java index 0973cab38ef0..724fe87dd83c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/HistogramAggregatorFactory.java @@ -202,7 +202,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { return "histogram"; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java index 8eb5963aa3a0..3332e959def7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SuppressedAggregatorFactory.java @@ -139,9 +139,9 @@ public List requiredFields() } @Override - public String getTypeName() + public String getComplexTypeName() { - return delegate.getTypeName(); + return delegate.getComplexTypeName(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java index aa119849d31e..4990d5d4b77f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java @@ -285,7 +285,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { return "hyperUnique"; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 2c7f07a9b4b8..15463cf1d938 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -283,7 +283,7 @@ public ValueType getType() @Override public ValueType getFinalizedType() { - // this is a copy of getTypeName in the hopes that someday groupby v1 is no more and it will report it's actual + // this is a copy of getComplexTypeName in the hopes that someday groupby v1 is no more and it will report it's actual // type of COMPLEX return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index d055cad0410e..9a45ac7ba7c1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -258,7 +258,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { return "serializablePairLongString"; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java index 89ecdd18e4e4..bdd3d59a5534 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java @@ -260,7 +260,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { if (isInputHyperUnique) { return "preComputedHyperUnique"; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 9eccf27e10dd..11a7e8d4b961 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -281,7 +281,7 @@ public ValueType getType() @Override public ValueType getFinalizedType() { - // this is a copy of getTypeName in the hopes that someday groupby v1 is no more and it will report it's actual + // this is a copy of getComplexTypeName in the hopes that someday groupby v1 is no more and it will report it's actual // type return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index 1c0373eb35b4..29543518f66c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -215,7 +215,7 @@ public byte[] getCacheKey() } @Override - public String getTypeName() + public String getComplexTypeName() { return "serializablePairLongString"; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java index 46172823d6cc..51ad33aaeaa7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/mean/DoubleMeanAggregatorFactory.java @@ -79,7 +79,7 @@ public List requiredFields() } @Override - public String getTypeName() + public String getComplexTypeName() { return "doubleMean"; } diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java index 487f0115f94f..e017a149f313 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java @@ -140,7 +140,7 @@ public ColumnValueSelector makeColumnValueSelector(final String column) // Wrap selector in a special one that uses ComplexMetricSerde to modify incoming objects. // For complex aggregators that read from multiple columns, we wrap all of them. This is not ideal but it // has worked so far. - final String complexTypeName = agg.getTypeName(); + final String complexTypeName = agg.getComplexTypeName(); final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(complexTypeName); if (serde == null) { throw new ISE("Don't know how to handle type[%s]", complexTypeName); @@ -1125,7 +1125,7 @@ public MetricDesc(int index, AggregatorFactory factory) } else { capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.COMPLEX) .setHasNulls(ColumnCapabilities.Capable.TRUE); - String complexTypeName = factory.getTypeName(); + String complexTypeName = factory.getComplexTypeName(); ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(complexTypeName); if (serde != null) { this.type = serde.getTypeName(); diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java index 32a3958a64d8..6f56e0df94b1 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java @@ -420,7 +420,7 @@ public ValueType getType() } @Override - public String getTypeName() + public String getComplexTypeName() { return TYPE; } From 9cbb602e88ca403dbbfc6ac764125ded25be49c4 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 19 Aug 2020 13:48:06 -0700 Subject: [PATCH 09/22] setup expression post agg for type inference existing --- .../druid/segment/column/ValueType.java | 19 +++++++++++-------- .../query/aggregation/PostAggregator.java | 4 ++++ .../post/ExpressionPostAggregator.java | 17 +++++++++++++---- .../virtual/ExpressionVirtualColumn.java | 3 ++- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/druid/segment/column/ValueType.java b/core/src/main/java/org/apache/druid/segment/column/ValueType.java index 7693f0963dfd..b839e1674ff9 100644 --- a/core/src/main/java/org/apache/druid/segment/column/ValueType.java +++ b/core/src/main/java/org/apache/druid/segment/column/ValueType.java @@ -30,9 +30,10 @@ * row signatures, and all other type needs. * * Currently only the primitive types ({@link #isPrimitive()} is true) and {@link #COMPLEX} can be stored in columns - * and are the only types handled directly by the query engines. Array types can currently be produced by expressions - * and by some post-aggregators, but do not currently have special engine handling, and should be used by implementors - * sparingly until full engine support is in place. + * and are also the only types handled directly by the query engines. Array types can currently be produced by + * expressions and by some post-aggregators, but do not currently have special engine handling, and should be used by + * implementors sparingly until full engine support is in place. Aggregators should never specify array types as their + * output type until the engines fully support these types. */ public enum ValueType { @@ -49,21 +50,23 @@ public enum ValueType STRING_ARRAY; + /** + * Type is a numeric type, not including numeric array types + */ public boolean isNumeric() { return isNumeric(this); } + /** + * Type is a 'primitive' type, which includes the {@link #isNumeric} types and {@link #STRING}, but not + * {@link #COMPLEX} or array types. + */ public boolean isPrimitive() { return this.equals(ValueType.STRING) || isNumeric(this); } - public boolean isComplex() - { - return !isPrimitive(); - } - @Nullable @JsonCreator public static ValueType fromString(@Nullable String name) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java index 010e7cc5e10f..542ed9bfb42c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java @@ -44,6 +44,10 @@ public interface PostAggregator extends Cacheable @Nullable String getName(); + /** + * Return the output {@link ValueType} of this post aggregator. + */ + @Nullable ValueType getType(); /** diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java index bd55a9f25283..fb47e388c42c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java @@ -64,6 +64,9 @@ public class ExpressionPostAggregator implements PostAggregator @Nullable private final String ordering; + @Nullable + private final ValueType outputType; + private final ExprMacroTable macroTable; private final Map> finalizers; @@ -104,6 +107,7 @@ private ExpressionPostAggregator( name, expression, ordering, + null, // in the future this will be computed by decorate macroTable, finalizers, parsed, @@ -115,6 +119,7 @@ private ExpressionPostAggregator( final String name, final String expression, @Nullable final String ordering, + @Nullable final ValueType outputType, final ExprMacroTable macroTable, final Map> finalizers, final Supplier parsed, @@ -126,6 +131,10 @@ private ExpressionPostAggregator( this.name = name; this.expression = expression; this.ordering = ordering; + // allow nulls to match previous behavior when type was never specified, however this should be non-nullable + // in the future, when expression support type inference + this.outputType = outputType; + // comparator should be specialized to output type ... someday this.comparator = ordering == null ? DEFAULT_COMPARATOR : Ordering.valueOf(ordering); this.macroTable = macroTable; this.finalizers = finalizers; @@ -172,9 +181,8 @@ public String getName() @Override public ValueType getType() { - // this is wrong, replace with Expr output type based on the input types once it is available - // but treat as string for now - return ValueType.STRING; + // computed by decorate + return outputType; } @Override @@ -184,6 +192,7 @@ public ExpressionPostAggregator decorate(final Map ag name, expression, ordering, + null, // this should be computed from expression output type once it supports output type inference macroTable, CollectionUtils.mapValues(aggregators, aggregatorFactory -> aggregatorFactory::finalizeComputation), parsed, @@ -229,7 +238,7 @@ public enum Ordering implements Comparator * Ensures the following order: numeric > NaN > Infinite. * * The name may be referenced via Ordering.valueOf(String) in the constructor {@link - * ExpressionPostAggregator#ExpressionPostAggregator(String, String, String, ExprMacroTable, Map, Supplier)}. + * ExpressionPostAggregator#ExpressionPostAggregator(String, String, String, ValueType, ExprMacroTable, Map, Supplier, Supplier)}. */ @SuppressWarnings("unused") numericFirst { diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index 59b846fa9a99..84a478a0de12 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -40,6 +40,7 @@ import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ValueType; +import javax.annotation.Nullable; import java.util.List; import java.util.Objects; @@ -54,7 +55,7 @@ public class ExpressionVirtualColumn implements VirtualColumn public ExpressionVirtualColumn( @JsonProperty("name") String name, @JsonProperty("expression") String expression, - @JsonProperty("outputType") ValueType outputType, + @JsonProperty("outputType") @Nullable ValueType outputType, @JacksonInject ExprMacroTable macroTable ) { From 0c8173515aee54e738a0a5bca0cb8a0b40810d07 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 19 Aug 2020 16:24:42 -0700 Subject: [PATCH 10/22] more javadocs --- .../druid/segment/column/ValueType.java | 56 ++++++++++++++++--- .../query/aggregation/AggregatorFactory.java | 5 +- .../query/aggregation/PostAggregator.java | 3 +- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/druid/segment/column/ValueType.java b/core/src/main/java/org/apache/druid/segment/column/ValueType.java index b839e1674ff9..866b086523f3 100644 --- a/core/src/main/java/org/apache/druid/segment/column/ValueType.java +++ b/core/src/main/java/org/apache/druid/segment/column/ValueType.java @@ -25,8 +25,8 @@ import javax.annotation.Nullable; /** - * This enumeration defines the Druid type system used to indicate the type of data stored in columns, produced by - * expressions, used to allow query processing engine algorithms to compute results, used to compute query result + * This enumeration defines the Druid type system used to indicate the type of data stored in columns and produced by + * expressions and aggregations, used to allow query processing engine algorithms to compute results, used to compute query result * row signatures, and all other type needs. * * Currently only the primitive types ({@link #isPrimitive()} is true) and {@link #COMPLEX} can be stored in columns @@ -37,17 +37,56 @@ */ public enum ValueType { - // primitive types + /** + * 64-bit double precision floating point number primitive type. This type may be used as a grouping key, or as an + * input to any aggregators which support primitive numerical operations like sums, minimums, maximums, etc, as well + * as an input to expression virtual columns. + */ DOUBLE, + /** + * 32-bit single precision floating point number primitive type. This type may be used as a grouping key, or as an + * input to any aggregators which support primitive numerical operations like sums, minimums, maximums, etc, as well + * as an input to expression virtual columns. + */ FLOAT, + /** + * 64-bit integer number primitve type. This type may be used as a grouping key, or as an + * input to any aggregators which support primitive numerical operations like sums, minimums, maximums, etc, as well + * as an input to expression virtual columns. + */ LONG, + /** + * String object type. This type may be used as a grouping key, an input to certain types of complex sketch + * aggregators, and as an input to expression virtual columns. String types might potentially be 'multi-valued' when + * stored in segments, and contextually at various layers of query processing, but this information is not available + * through this enum alone, and must be accompany this type indicator to properly handle. + */ STRING, - // non-primitive types - COMPLEX, - // transient array types (also non-primitive) + /** + * Array object of 64-bit double precision floating point numbers. This type is not currently supported as a grouping + * key for aggregations, cannot be used as an input for numerical primitive aggregations such as sums, and may have + * limited support as an input among complex type sketch aggregators. + */ DOUBLE_ARRAY, + /** + * Array object of 64-bit integer numbers. This type is not currently supported as a grouping key for aggregations, + * and may have limited support as an input among complex type sketch aggregators. + */ LONG_ARRAY, - STRING_ARRAY; + /** + * Array object of String objects. This type is not currently supported as a grouping key for aggregations, + * and may have limited support as an input among complex type sketch aggregators. + */ + STRING_ARRAY, + /** + * Placeholder for arbitrary 'complex' types, which have a corresponding serializer/deserializer implementation. Note + * that knowing a type is complex alone isn't enough information to work with it directly, and additional information + * in the form of a type name that is registered in the complex type registry must be available to make this type + * meaningful. This type is not currently supported as a grouping key for aggregations, and may not be used as an + * input to expression virtual columns, and might only be supported by the specific aggregators crafted to handle + * this complex type. + */ + COMPLEX; /** @@ -61,6 +100,9 @@ public boolean isNumeric() /** * Type is a 'primitive' type, which includes the {@link #isNumeric} types and {@link #STRING}, but not * {@link #COMPLEX} or array types. + * + * Primitive types support being used for grouping to compute aggregates in both group by and top-n query engines, + * while non-primitive types currently do not */ public boolean isPrimitive() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java index 3d5d2e4a81ff..2acbe66dc325 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorFactory.java @@ -216,6 +216,8 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * Get the "intermediate" {@link ValueType} for this aggregator. This is the same as the type returned by * {@link #deserialize} and the type accepted by {@link #combine}. However, it is *not* necessarily the same type * returned by {@link #finalizeComputation}. + * + * Refer to the {@link ValueType} javadocs for details on the implications of choosing a type. */ public abstract ValueType getType(); @@ -223,7 +225,8 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre * Get the type for the final form of this this aggregator, i.e. the type of the value returned by * {@link #finalizeComputation}. This may be the same as or different than the types expected in {@link #deserialize} * and {@link #combine}. - * @return + * + * Refer to the {@link ValueType} javadocs for details on the implications of choosing a type. */ public abstract ValueType getFinalizedType(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java index 542ed9bfb42c..5d58b75b9a71 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java @@ -45,7 +45,8 @@ public interface PostAggregator extends Cacheable String getName(); /** - * Return the output {@link ValueType} of this post aggregator. + * Return the output type of a row processed with this post aggregator. Refer to the {@link ValueType} javadocs + * for details on the implications of choosing a type. */ @Nullable ValueType getType(); From c4371c86ddfa8a8b478b1dbaff453410cdf1f307 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 24 Aug 2020 13:58:29 -0700 Subject: [PATCH 11/22] fixup --- .../datasketches/hll/HllSketchToEstimatePostAggregator.java | 2 +- .../quantiles/DoublesSketchToHistogramPostAggregator.java | 2 +- .../datasketches/theta/SketchEstimatePostAggregator.java | 2 +- .../query/aggregation/post/FieldAccessPostAggregator.java | 4 ++-- .../aggregation/post/FinalizingFieldAccessPostAggregator.java | 4 +++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java index e11daa5fd667..0f058d6f0d51 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregator.java @@ -66,7 +66,7 @@ public String getName() @Override public ValueType getType() { - return ValueType.DOUBLE; + return round ? ValueType.LONG : ValueType.DOUBLE; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java index fe0e7c4e4fdb..5cc5cb7267d7 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregator.java @@ -109,7 +109,7 @@ public String getName() @Override public ValueType getType() { - return ValueType.COMPLEX; + return ValueType.DOUBLE_ARRAY; } @JsonProperty diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java index fb27e6a96f4f..0474c9897b47 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java @@ -104,7 +104,7 @@ public String getName() @Override public ValueType getType() { - return ValueType.DOUBLE; + return errorBoundsStdDev ? ValueType.COMPLEX : ValueType.DOUBLE; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java index 2c129583cfa4..a3c349a36cf0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -50,10 +50,10 @@ public FieldAccessPostAggregator( @JsonProperty("fieldName") String fieldName ) { - this(name, fieldName, ValueTypes.defaultAggregationType()); + this(name, fieldName, null); } - private FieldAccessPostAggregator(@Nullable String name, String fieldName, ValueType type) + private FieldAccessPostAggregator(@Nullable String name, String fieldName, @Nullable ValueType type) { Preconditions.checkNotNull(fieldName); this.name = name; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java index f370d50c3170..7c24bcdf7644 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java @@ -29,6 +29,7 @@ import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueTypes; +import javax.annotation.Nullable; import java.util.Comparator; import java.util.Map; import java.util.Objects; @@ -39,6 +40,7 @@ public class FinalizingFieldAccessPostAggregator implements PostAggregator { private final String name; private final String fieldName; + @Nullable private final ValueType finalizedType; private final Comparator comparator; private final Function finalizer; @@ -55,7 +57,7 @@ public FinalizingFieldAccessPostAggregator( private FinalizingFieldAccessPostAggregator( final String name, final String fieldName, - final ValueType finalizedType, + @Nullable final ValueType finalizedType, final Comparator comparator, final Function finalizer ) From 1e686d80c846aadfc2aee57966d1e2ed9b070cb4 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 24 Aug 2020 13:59:21 -0700 Subject: [PATCH 12/22] oops --- .../datasketches/theta/SketchEstimatePostAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java index 0474c9897b47..4f970e807872 100644 --- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java +++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java @@ -104,7 +104,7 @@ public String getName() @Override public ValueType getType() { - return errorBoundsStdDev ? ValueType.COMPLEX : ValueType.DOUBLE; + return errorBoundsStdDev != null ? ValueType.COMPLEX : ValueType.DOUBLE; } @Override From 881954fc59d5dc219a962e4d25ea41a659fa550e Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 24 Aug 2020 14:07:16 -0700 Subject: [PATCH 13/22] more test --- ...HllSketchToEstimatePostAggregatorTest.java | 51 +++++++++++++++++++ ...esSketchToHistogramPostAggregatorTest.java | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregatorTest.java index d8facd1e2f22..eb3a4660ae98 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchToEstimatePostAggregatorTest.java @@ -22,8 +22,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.Druids; +import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -70,4 +77,48 @@ public void testEqualsAndHashCode() .usingGetClass() .verify(); } + + @Test + public void testResultArraySignature() + { + final TimeseriesQuery query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2000/3000") + .granularity(Granularities.HOUR) + .aggregators( + new CountAggregatorFactory("count"), + new HllSketchMergeAggregatorFactory( + "hllMerge", + "col", + null, + null, + false + ) + ) + .postAggregators( + new HllSketchToEstimatePostAggregator( + "hllEstimate", + new FieldAccessPostAggregator(null, "hllMerge"), + false + ), + new HllSketchToEstimatePostAggregator( + "hllEstimateRound", + new FieldAccessPostAggregator(null, "hllMerge"), + true + ) + ) + .build(); + + Assert.assertEquals( + RowSignature.builder() + .addTimeColumn() + .add("count", ValueType.LONG) + .add("hllMerge", null) + .add("hllEstimate", ValueType.DOUBLE) + .add("hllEstimateRound", ValueType.LONG) + .build(), + new TimeseriesQueryQueryToolChest().resultArraySignature(query) + ); + } } diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java index 3163bfc776b3..48261f76f996 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchToHistogramPostAggregatorTest.java @@ -212,7 +212,7 @@ public void testResultArraySignature() RowSignature.builder() .addTimeColumn() .add("sketch", null) - .add("a", ValueType.COMPLEX) + .add("a", ValueType.DOUBLE_ARRAY) .build(), new TimeseriesQueryQueryToolChest().resultArraySignature(query) ); From dc27526a18f3378f15031f035dee9b90f54a832a Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 24 Aug 2020 14:14:15 -0700 Subject: [PATCH 14/22] more test --- .../theta/SketchAggregatorFactoryTest.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java index af31183e43aa..9abec0728004 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregatorFactoryTest.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.datasketches.theta; +import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.Druids; import org.apache.druid.query.aggregation.CountAggregatorFactory; @@ -61,7 +62,30 @@ public void testResultArraySignature() new FieldAccessPostAggregator("merge-access", "merge"), new FinalizingFieldAccessPostAggregator("merge-finalize", "merge"), new FieldAccessPostAggregator("mergeFinalize-access", "mergeFinalize"), - new FinalizingFieldAccessPostAggregator("mergeFinalize-finalize", "mergeFinalize") + new FinalizingFieldAccessPostAggregator("mergeFinalize-finalize", "mergeFinalize"), + new SketchEstimatePostAggregator( + "sketchEstimate", + new FieldAccessPostAggregator(null, "merge"), + null + ), + new SketchEstimatePostAggregator( + "sketchEstimateStdDev", + new FieldAccessPostAggregator(null, "merge"), + 2 + ), + new SketchSetPostAggregator( + "sketchSet", + "UNION", + null, + ImmutableList.of( + new FieldAccessPostAggregator(null, "oldMerge"), + new FieldAccessPostAggregator(null, "merge") + ) + ), + new SketchToStringPostAggregator( + "sketchString", + new FieldAccessPostAggregator(null, "merge") + ) ) .build(); @@ -84,6 +108,10 @@ public void testResultArraySignature() .add("merge-finalize", ValueType.COMPLEX) .add("mergeFinalize-access", ValueType.COMPLEX) .add("mergeFinalize-finalize", ValueType.DOUBLE) + .add("sketchEstimate", ValueType.DOUBLE) + .add("sketchEstimateStdDev", ValueType.COMPLEX) + .add("sketchSet", ValueType.COMPLEX) + .add("sketchString", ValueType.STRING) .build(), new TimeseriesQueryQueryToolChest().resultArraySignature(query) ); From 6eee25a2579eae9f6450a0a4ce315006ee6027a0 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 24 Aug 2020 21:24:27 -0700 Subject: [PATCH 15/22] more comments/javadoc --- .../apache/druid/query/aggregation/PostAggregator.java | 9 ++++----- .../query/aggregation/post/ExpressionPostAggregator.java | 2 ++ .../aggregation/post/FieldAccessPostAggregator.java | 2 ++ .../post/FinalizingFieldAccessPostAggregator.java | 2 ++ 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java index 5d58b75b9a71..2996a18197e4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/PostAggregator.java @@ -52,11 +52,10 @@ public interface PostAggregator extends Cacheable ValueType getType(); /** - * Returns a richer post aggregator which are built from the given aggregators with their names and some accessible - * environmental variables such as ones in the object scope. - * - * @param aggregators A map of aggregator factories with their names. - * + * Allows returning an enriched post aggregator, built from contextual information available from the given map of + * {@link AggregatorFactory} keyed by their names. Callers must call this method before calling {@link #compute} or + * {@link #getComparator}. This is typically done in the constructor of queries which support post aggregators, via + * {@link org.apache.druid.query.Queries#prepareAggregations}. */ PostAggregator decorate(Map aggregators); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java index fb47e388c42c..978bf3e04abc 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/ExpressionPostAggregator.java @@ -64,6 +64,8 @@ public class ExpressionPostAggregator implements PostAggregator @Nullable private final String ordering; + // type is ignored from equals and friends because it is computed by decorate, and all post-aggs should be decorated + // prior to usage (and is currently done so in the query constructors of all queries which can have post-aggs) @Nullable private final ValueType outputType; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java index a3c349a36cf0..3af3286e9a7e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -41,6 +41,8 @@ public class FieldAccessPostAggregator implements PostAggregator @Nullable private final String name; private final String fieldName; + // type is ignored from equals and friends because it is computed by decorate, and all post-aggs should be decorated + // prior to usage (and is currently done so in the query constructors of all queries which can have post-aggs) @Nullable private final ValueType type; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java index 7c24bcdf7644..e92cb709727a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java @@ -40,6 +40,8 @@ public class FinalizingFieldAccessPostAggregator implements PostAggregator { private final String name; private final String fieldName; + // type is ignored from equals and friends because it is computed by decorate, and all post-aggs should be decorated + // prior to usage (and is currently done so in the query constructors of all queries which can have post-aggs) @Nullable private final ValueType finalizedType; private final Comparator comparator; From 0c7ba70f4c768d27d2f34f1c3ea8fb779025a5ab Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2020 04:13:55 -0700 Subject: [PATCH 16/22] nulls --- .../hyperloglog/HyperUniqueFinalizingPostAggregator.java | 3 +-- .../query/aggregation/post/FieldAccessPostAggregator.java | 3 +-- .../post/FinalizingFieldAccessPostAggregator.java | 3 +-- .../java/org/apache/druid/segment/column/ValueTypes.java | 5 ----- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java index 8529f43b07ff..103917678c03 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/hyperloglog/HyperUniqueFinalizingPostAggregator.java @@ -29,7 +29,6 @@ import org.apache.druid.query.aggregation.post.PostAggregatorIds; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; import java.util.Comparator; @@ -109,7 +108,7 @@ public ValueType getType() { return aggregatorFactory != null ? aggregatorFactory.getFinalizedType() - : ValueTypes.defaultAggregationType(); + : null; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java index 3af3286e9a7e..23c8ee9418d9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -27,7 +27,6 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; import java.util.Comparator; @@ -103,7 +102,7 @@ public FieldAccessPostAggregator decorate(Map aggrega if (aggregators != null && aggregators.containsKey(fieldName)) { type = aggregators.get(fieldName).getType(); } else { - type = ValueTypes.defaultAggregationType(); + type = null; } return new FieldAccessPostAggregator( diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java index e92cb709727a..4acf45d9f648 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregator.java @@ -27,7 +27,6 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.column.ValueTypes; import javax.annotation.Nullable; import java.util.Comparator; @@ -126,7 +125,7 @@ public FinalizingFieldAccessPostAggregator decorate(final Map Date: Tue, 25 Aug 2020 12:37:12 -0700 Subject: [PATCH 17/22] explicitly handle only numeric and complex aggregators for incremental index --- .../druid/segment/incremental/IncrementalIndex.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java index e017a149f313..6832698660ac 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java @@ -1119,10 +1119,10 @@ public MetricDesc(int index, AggregatorFactory factory) ValueType valueType = factory.getType(); - if (valueType.isPrimitive()) { + if (valueType.isNumeric()) { capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); this.type = valueType.toString(); - } else { + } else if (ValueType.COMPLEX.equals(valueType)){ capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.COMPLEX) .setHasNulls(ColumnCapabilities.Capable.TRUE); String complexTypeName = factory.getComplexTypeName(); @@ -1130,8 +1130,12 @@ public MetricDesc(int index, AggregatorFactory factory) if (serde != null) { this.type = serde.getTypeName(); } else { - throw new ISE("Don't know how to handle complex type[%s] of type[%s]", complexTypeName, valueType); + throw new ISE("Unable to handle complex type[%s] of type[%s]", complexTypeName, valueType); } + } else { + // if we need to handle non-numeric and non-complex types (e.g. strings, arrays) it should be done here + // and we should determine the appropriate ColumnCapabilities + throw new ISE("Unable to handle type[%s] for AggregatorFactory[%s]", valueType, factory.getClass()); } } From 59515b488bd845ee198fa14fc98aa4f2bba08e1c Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2020 13:46:03 -0700 Subject: [PATCH 18/22] checkstyle --- .../org/apache/druid/segment/incremental/IncrementalIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java index 6832698660ac..437d30e273e3 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndex.java @@ -1122,7 +1122,7 @@ public MetricDesc(int index, AggregatorFactory factory) if (valueType.isNumeric()) { capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); this.type = valueType.toString(); - } else if (ValueType.COMPLEX.equals(valueType)){ + } else if (ValueType.COMPLEX.equals(valueType)) { capabilities = ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.COMPLEX) .setHasNulls(ColumnCapabilities.Capable.TRUE); String complexTypeName = factory.getComplexTypeName(); From 0d709aceee41deaecb64ffa15e4c9865732f6294 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2020 14:53:07 -0700 Subject: [PATCH 19/22] more tests --- .../druid/segment/column/ValueType.java | 13 +++++ .../RowBasedColumnSelectorFactory.java | 11 +++- .../druid/query/InlineDataSourceTest.java | 15 ++++-- .../RowBasedColumnSelectorFactoryTest.java | 51 ++++++++++++++++++- 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/apache/druid/segment/column/ValueType.java b/core/src/main/java/org/apache/druid/segment/column/ValueType.java index 866b086523f3..fa5e4894ef2f 100644 --- a/core/src/main/java/org/apache/druid/segment/column/ValueType.java +++ b/core/src/main/java/org/apache/druid/segment/column/ValueType.java @@ -97,6 +97,14 @@ public boolean isNumeric() return isNumeric(this); } + /** + * Type is an array type + */ + public boolean isArray() + { + return isArray(this); + } + /** * Type is a 'primitive' type, which includes the {@link #isNumeric} types and {@link #STRING}, but not * {@link #COMPLEX} or array types. @@ -123,4 +131,9 @@ public static boolean isNumeric(ValueType type) { return type == ValueType.LONG || type == ValueType.FLOAT || type == ValueType.DOUBLE; } + + public static boolean isArray(ValueType type) + { + return type == ValueType.DOUBLE_ARRAY || type == ValueType.LONG_ARRAY || type == ValueType.STRING_ARRAY; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java index b608cbcb6735..e3130e4ccd3a 100644 --- a/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java @@ -102,15 +102,22 @@ static ColumnCapabilities getColumnCapabilities( // Do _not_ set isDictionaryEncoded or hasBitmapIndexes, because Row-based columns do not have those things. // Do not set hasMultipleValues, because even though we might return multiple values, setting it affirmatively - // causes expression selectors to always treat us as arrays, so leave as unknown. + // causes expression selectors to always treat us as arrays, so leave as unknown, unless we know it is explictly + // an array type if (valueType != null) { if (valueType.isNumeric()) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); } - return new ColumnCapabilitiesImpl() + + ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl() .setType(valueType) .setDictionaryValuesUnique(false) .setDictionaryValuesSorted(false); + + if (valueType.isArray()) { + capabilities.setHasMultipleValues(true); + } + return capabilities; } else { return null; } diff --git a/processing/src/test/java/org/apache/druid/query/InlineDataSourceTest.java b/processing/src/test/java/org/apache/druid/query/InlineDataSourceTest.java index 9fa4656a2753..d1f8af89ec71 100644 --- a/processing/src/test/java/org/apache/druid/query/InlineDataSourceTest.java +++ b/processing/src/test/java/org/apache/druid/query/InlineDataSourceTest.java @@ -46,10 +46,11 @@ public class InlineDataSourceTest private final AtomicLong iterationCounter = new AtomicLong(); + // raw arrays, e.g. double[]{1.0, 2.0} on round trip get deserialized into List, so just model that private final List rows = ImmutableList.of( - new Object[]{DateTimes.of("2000").getMillis(), "foo", 0d, ImmutableMap.of("n", "0")}, - new Object[]{DateTimes.of("2000").getMillis(), "bar", 1d, ImmutableMap.of("n", "1")}, - new Object[]{DateTimes.of("2000").getMillis(), "baz", 2d, ImmutableMap.of("n", "2")} + new Object[]{DateTimes.of("2000").getMillis(), "foo", 0d, ImmutableMap.of("n", "0"), ImmutableList.of(1.0, 2.0)}, + new Object[]{DateTimes.of("2000").getMillis(), "bar", 1d, ImmutableMap.of("n", "1"), ImmutableList.of(2.0, 4.0)}, + new Object[]{DateTimes.of("2000").getMillis(), "baz", 2d, ImmutableMap.of("n", "2"), ImmutableList.of(3.0, 6.0)} ); private final Iterable rowsIterable = () -> { @@ -61,14 +62,16 @@ public class InlineDataSourceTest ColumnHolder.TIME_COLUMN_NAME, "str", "double", - "complex" + "complex", + "double_array" ); private final List expectedColumnTypes = ImmutableList.of( ValueType.LONG, ValueType.STRING, ValueType.DOUBLE, - ValueType.COMPLEX + ValueType.COMPLEX, + ValueType.DOUBLE_ARRAY ); private final RowSignature expectedRowSignature; @@ -127,6 +130,7 @@ public void test_getRowSignature() .add("str", ValueType.STRING) .add("double", ValueType.DOUBLE) .add("complex", ValueType.COMPLEX) + .add("double_array", ValueType.DOUBLE_ARRAY) .build(), listDataSource.getRowSignature() ); @@ -160,6 +164,7 @@ public void test_rowAdapter() Assert.assertEquals("bar", adapter.columnFunction("str").apply(row)); Assert.assertEquals(1d, adapter.columnFunction("double").apply(row)); Assert.assertEquals(ImmutableMap.of("n", "1"), adapter.columnFunction("complex").apply(row)); + Assert.assertEquals(ImmutableList.of(2.0, 4.0), adapter.columnFunction("double_array").apply(row)); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/RowBasedColumnSelectorFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/RowBasedColumnSelectorFactoryTest.java index a802b819b570..26967d5b1c5d 100644 --- a/processing/src/test/java/org/apache/druid/segment/RowBasedColumnSelectorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/RowBasedColumnSelectorFactoryTest.java @@ -23,16 +23,20 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; -public class RowBasedColumnSelectorFactoryTest +public class RowBasedColumnSelectorFactoryTest extends InitializedNullHandlingTest { private static final String STRING_COLUMN_NAME = "string"; private static final String LONG_COLUMN_NAME = "long"; private static final String FLOAT_COLUMN_NAME = "float"; private static final String DOUBLE_COLUMN_NAME = "double"; private static final String COMPLEX_COLUMN_NAME = "complex"; + private static final String DOUBLE_ARRAY_COLUMN_NAME = "double_array"; + private static final String LONG_ARRAY_COLUMN_NAME = "long_array"; + private static final String STRING_ARRAY_COLUMN_NAME = "string_array"; private static final RowSignature ROW_SIGNATURE = RowSignature.builder() .add(ColumnHolder.TIME_COLUMN_NAME, ValueType.LONG) @@ -41,6 +45,9 @@ public class RowBasedColumnSelectorFactoryTest .add(FLOAT_COLUMN_NAME, ValueType.FLOAT) .add(DOUBLE_COLUMN_NAME, ValueType.DOUBLE) .add(COMPLEX_COLUMN_NAME, ValueType.COMPLEX) + .add(DOUBLE_ARRAY_COLUMN_NAME, ValueType.DOUBLE_ARRAY) + .add(LONG_ARRAY_COLUMN_NAME, ValueType.LONG_ARRAY) + .add(STRING_ARRAY_COLUMN_NAME, ValueType.STRING_ARRAY) .build(); @Test @@ -128,6 +135,48 @@ public void testCapabilitiesComplex() Assert.assertFalse(caps.hasSpatialIndexes()); } + @Test + public void testCapabilitiesDoubleArray() + { + ColumnCapabilities caps = + RowBasedColumnSelectorFactory.getColumnCapabilities(ROW_SIGNATURE, DOUBLE_ARRAY_COLUMN_NAME); + Assert.assertEquals(ValueType.DOUBLE_ARRAY, caps.getType()); + Assert.assertFalse(caps.hasBitmapIndexes()); + Assert.assertFalse(caps.isDictionaryEncoded().isTrue()); + Assert.assertFalse(caps.areDictionaryValuesSorted().isTrue()); + Assert.assertFalse(caps.areDictionaryValuesUnique().isTrue()); + Assert.assertTrue(caps.hasMultipleValues().isTrue()); + Assert.assertFalse(caps.hasSpatialIndexes()); + } + + @Test + public void testCapabilitiesLongArray() + { + ColumnCapabilities caps = + RowBasedColumnSelectorFactory.getColumnCapabilities(ROW_SIGNATURE, LONG_ARRAY_COLUMN_NAME); + Assert.assertEquals(ValueType.LONG_ARRAY, caps.getType()); + Assert.assertFalse(caps.hasBitmapIndexes()); + Assert.assertFalse(caps.isDictionaryEncoded().isTrue()); + Assert.assertFalse(caps.areDictionaryValuesSorted().isTrue()); + Assert.assertFalse(caps.areDictionaryValuesUnique().isTrue()); + Assert.assertTrue(caps.hasMultipleValues().isTrue()); + Assert.assertFalse(caps.hasSpatialIndexes()); + } + + @Test + public void testCapabilitiesStringArray() + { + ColumnCapabilities caps = + RowBasedColumnSelectorFactory.getColumnCapabilities(ROW_SIGNATURE, STRING_ARRAY_COLUMN_NAME); + Assert.assertEquals(ValueType.STRING_ARRAY, caps.getType()); + Assert.assertFalse(caps.hasBitmapIndexes()); + Assert.assertFalse(caps.isDictionaryEncoded().isTrue()); + Assert.assertFalse(caps.areDictionaryValuesSorted().isTrue()); + Assert.assertFalse(caps.areDictionaryValuesUnique().isTrue()); + Assert.assertTrue(caps.hasMultipleValues().isTrue()); + Assert.assertFalse(caps.hasSpatialIndexes()); + } + @Test public void testCapabilitiesUnknownColumn() { From 207f44846b80c3ebd1bb4d245c200792100d1ad2 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2020 14:58:27 -0700 Subject: [PATCH 20/22] adjust --- .../RowBasedColumnSelectorFactory.java | 19 +++++++++---------- .../column/ColumnCapabilitiesImpl.java | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java index e3130e4ccd3a..bfbdc531ec58 100644 --- a/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/RowBasedColumnSelectorFactory.java @@ -100,24 +100,23 @@ static ColumnCapabilities getColumnCapabilities( } else { final ValueType valueType = rowSignature.getColumnType(columnName).orElse(null); - // Do _not_ set isDictionaryEncoded or hasBitmapIndexes, because Row-based columns do not have those things. - // Do not set hasMultipleValues, because even though we might return multiple values, setting it affirmatively - // causes expression selectors to always treat us as arrays, so leave as unknown, unless we know it is explictly - // an array type + if (valueType != null) { if (valueType.isNumeric()) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); } - ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl() + if (valueType.isArray()) { + return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(valueType); + } + + // Do _not_ set isDictionaryEncoded or hasBitmapIndexes, because Row-based columns do not have those things. + // Do not set hasMultipleValues, because even though we might return multiple values, setting it affirmatively + // causes expression selectors to always treat us as arrays, so leave as unknown + return new ColumnCapabilitiesImpl() .setType(valueType) .setDictionaryValuesUnique(false) .setDictionaryValuesSorted(false); - - if (valueType.isArray()) { - capabilities.setHasMultipleValues(true); - } - return capabilities; } else { return null; } diff --git a/processing/src/main/java/org/apache/druid/segment/column/ColumnCapabilitiesImpl.java b/processing/src/main/java/org/apache/druid/segment/column/ColumnCapabilitiesImpl.java index f389adb02176..6348977fa56b 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ColumnCapabilitiesImpl.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ColumnCapabilitiesImpl.java @@ -168,6 +168,22 @@ public static ColumnCapabilitiesImpl createSimpleNumericColumnCapabilities(Value return builder; } + /** + * Similar to {@link #createSimpleNumericColumnCapabilities} except {@link #hasMultipleValues} is explicitly true + * and {@link #hasNulls} is not set + */ + public static ColumnCapabilitiesImpl createSimpleArrayColumnCapabilities(ValueType valueType) + { + ColumnCapabilitiesImpl builder = new ColumnCapabilitiesImpl().setType(valueType) + .setHasMultipleValues(true) + .setHasBitmapIndexes(false) + .setDictionaryEncoded(false) + .setDictionaryValuesSorted(false) + .setDictionaryValuesUnique(false) + .setHasSpatialIndexes(false); + return builder; + } + @Nullable private ValueType type = null; From 10e2f3ef1f4ed62c6c4445cf45a8f063c2552642 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2020 18:22:44 -0700 Subject: [PATCH 21/22] more tests to showcase difference in behavior --- .../server/ClientQuerySegmentWalkerTest.java | 409 +++++++++++++++++- 1 file changed, 403 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java index 2d2128edb4f3..89364c1ed7dd 100644 --- a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java @@ -119,6 +119,8 @@ public class ClientQuerySegmentWalkerTest private static final String BAR = "bar"; private static final String MULTI = "multi"; private static final String GLOBAL = "broadcast"; + private static final String ARRAY = "array"; + private static final String ARRAY_UNKNOWN = "array_nulls"; private static final Interval INTERVAL = Intervals.of("2000/P1Y"); private static final String VERSION = "A"; @@ -166,6 +168,40 @@ public class ClientQuerySegmentWalkerTest .build() ); + + private static final List ARRAY_INLINE_ROWS = + ImmutableList.builder() + .add(new Object[]{INTERVAL.getStartMillis(), "x", 1, ImmutableList.of(1.0, 2.0), ImmutableList.of(1L, 2L), ImmutableList.of("1.0", "2.0")}) + .add(new Object[]{INTERVAL.getStartMillis(), "x", 2, ImmutableList.of(2.0, 4.0), ImmutableList.of(2L, 4L), ImmutableList.of("2.0", "4.0")}) + .add(new Object[]{INTERVAL.getStartMillis(), "y", 3, ImmutableList.of(3.0, 6.0), ImmutableList.of(3L, 6L), ImmutableList.of("3.0", "6.0")}) + .add(new Object[]{INTERVAL.getStartMillis(), "z", 4, ImmutableList.of(4.0, 8.0), ImmutableList.of(4L, 8L), ImmutableList.of("4.0", "8.0")}) + .build(); + + private static final InlineDataSource ARRAY_INLINE = InlineDataSource.fromIterable( + ARRAY_INLINE_ROWS, + RowSignature.builder() + .addTimeColumn() + .add("s", ValueType.STRING) + .add("n", ValueType.LONG) + .add("ad", ValueType.DOUBLE_ARRAY) + .add("al", ValueType.LONG_ARRAY) + .add("as", ValueType.STRING_ARRAY) + .build() + ); + + + private static final InlineDataSource ARRAY_INLINE_UNKNOWN = InlineDataSource.fromIterable( + ARRAY_INLINE_ROWS, + RowSignature.builder() + .addTimeColumn() + .add("s", ValueType.STRING) + .add("n", ValueType.LONG) + .add("ad", null) + .add("al", null) + .add("as", null) + .build() + ); + @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -635,6 +671,365 @@ public void testTimeseriesOnGroupByOnTableErrorTooManyRows() testQuery(query, ImmutableList.of(), ImmutableList.of()); } + @Test + public void testGroupByOnArraysDoubles() + { + final GroupByQuery query = + (GroupByQuery) GroupByQuery.builder() + .setDataSource(ARRAY) + .setGranularity(Granularities.ALL) + .setInterval(Collections.singletonList(INTERVAL)) + .setDimensions(DefaultDimensionSpec.of("ad")) + .build() + .withId("queryId"); + + + // group by cannot handle true array types, expect this, RuntimeExeception with IAE in stack trace + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [DOUBLE_ARRAY]"); + + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of() + ); + } + + @Test + public void testGroupByOnArraysUnknownDoubles() + { + final GroupByQuery query = + (GroupByQuery) GroupByQuery.builder() + .setDataSource(ARRAY_UNKNOWN) + .setGranularity(Granularities.ALL) + .setInterval(Collections.singletonList(INTERVAL)) + .setDimensions(DefaultDimensionSpec.of("ad")) + .build() + .withId("queryId"); + + + // 'unknown' is treated as ValueType.STRING. this might not always be the case, so this is a test case of wacky + // behavior of sorts + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of( + new Object[]{"1.0"}, + new Object[]{"2.0"}, + new Object[]{"3.0"}, + new Object[]{"4.0"}, + new Object[]{"6.0"}, + new Object[]{"8.0"} + ) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + + @Test + public void testGroupByOnArraysLongs() + { + final GroupByQuery query = + (GroupByQuery) GroupByQuery.builder() + .setDataSource(ARRAY) + .setGranularity(Granularities.ALL) + .setInterval(Collections.singletonList(INTERVAL)) + .setDimensions(DefaultDimensionSpec.of("al")) + .build() + .withId("queryId"); + + // group by cannot handle true array types, expect this, RuntimeExeception with IAE in stack trace + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [LONG_ARRAY]"); + + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of() + ); + } + + @Test + public void testGroupByOnArraysUnknownLongs() + { + final GroupByQuery query = + (GroupByQuery) GroupByQuery.builder() + .setDataSource(ARRAY_UNKNOWN) + .setGranularity(Granularities.ALL) + .setInterval(Collections.singletonList(INTERVAL)) + .setDimensions(DefaultDimensionSpec.of("al")) + .build() + .withId("queryId"); + + + // 'unknown' is treated as ValueType.STRING. this might not always be the case, so this is a test case of wacky + // behavior of sorts + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of( + new Object[]{"1"}, + new Object[]{"2"}, + new Object[]{"3"}, + new Object[]{"4"}, + new Object[]{"6"}, + new Object[]{"8"} + ) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + + @Test + public void testGroupByOnArraysStrings() + { + final GroupByQuery query = + (GroupByQuery) GroupByQuery.builder() + .setDataSource(ARRAY) + .setGranularity(Granularities.ALL) + .setInterval(Collections.singletonList(INTERVAL)) + .setDimensions(DefaultDimensionSpec.of("as")) + .build() + .withId("queryId"); + + + // group by cannot handle true array types, expect this, RuntimeExeception with IAE in stack trace + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [STRING_ARRAY]"); + + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of() + ); + } + + @Test + public void testGroupByOnArraysUnknownStrings() + { + final GroupByQuery query = + (GroupByQuery) GroupByQuery.builder() + .setDataSource(ARRAY_UNKNOWN) + .setGranularity(Granularities.ALL) + .setInterval(Collections.singletonList(INTERVAL)) + .setDimensions(DefaultDimensionSpec.of("as")) + .build() + .withId("queryId"); + + + // 'unknown' is treated as ValueType.STRING. this might not always be the case, so this is a test case of wacky + // behavior of sorts + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of( + new Object[]{"1.0"}, + new Object[]{"2.0"}, + new Object[]{"3.0"}, + new Object[]{"4.0"}, + new Object[]{"6.0"}, + new Object[]{"8.0"} + ) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + + + @Test + public void testTopNArraysDoubles() + { + final TopNQuery query = + (TopNQuery) new TopNQueryBuilder().dataSource(ARRAY) + .granularity(Granularities.ALL) + .intervals(Intervals.ONLY_ETERNITY) + .dimension(DefaultDimensionSpec.of("ad")) + .metric("sum_n") + .threshold(1000) + .aggregators(new LongSumAggregatorFactory("sum_n", "n")) + .build() + .withId(UUID.randomUUID().toString()); + + + // group by cannot handle true array types, expect this, RuntimeExeception with IAE in stack trace + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [DOUBLE_ARRAY]"); + + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of() + ); + } + + @Test + public void testTopNOnArraysUnknownDoubles() + { + final TopNQuery query = + (TopNQuery) new TopNQueryBuilder().dataSource(ARRAY_UNKNOWN) + .granularity(Granularities.ALL) + .intervals(Intervals.ONLY_ETERNITY) + .dimension(DefaultDimensionSpec.of("ad")) + .metric("sum_n") + .threshold(1000) + .aggregators(new LongSumAggregatorFactory("sum_n", "n")) + .build() + .withId(UUID.randomUUID().toString()); + + + // 'unknown' is treated as ValueType.STRING. this might not always be the case, so this is a test case of wacky + // behavior of sorts + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of( + new Object[]{946684800000L, "4.0", 6L}, + new Object[]{946684800000L, "8.0", 4L}, + new Object[]{946684800000L, "2.0", 3L}, + new Object[]{946684800000L, "3.0", 3L}, + new Object[]{946684800000L, "6.0", 3L}, + new Object[]{946684800000L, "1.0", 1L} + ) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + + @Test + public void testTopNOnArraysLongs() + { + final TopNQuery query = + (TopNQuery) new TopNQueryBuilder().dataSource(ARRAY) + .granularity(Granularities.ALL) + .intervals(Intervals.ONLY_ETERNITY) + .dimension(DefaultDimensionSpec.of("al")) + .metric("sum_n") + .threshold(1000) + .aggregators(new LongSumAggregatorFactory("sum_n", "n")) + .build() + .withId(UUID.randomUUID().toString()); + + // group by cannot handle true array types, expect this, RuntimeExeception with IAE in stack trace + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [LONG_ARRAY]"); + + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of() + ); + } + + @Test + public void testTopNOnArraysUnknownLongs() + { + final TopNQuery query = + (TopNQuery) new TopNQueryBuilder().dataSource(ARRAY_UNKNOWN) + .granularity(Granularities.ALL) + .intervals(Intervals.ONLY_ETERNITY) + .dimension(DefaultDimensionSpec.of("al")) + .metric("sum_n") + .threshold(1000) + .aggregators(new LongSumAggregatorFactory("sum_n", "n")) + .build() + .withId(UUID.randomUUID().toString()); + + + // 'unknown' is treated as ValueType.STRING. this might not always be the case, so this is a test case of wacky + // behavior of sorts + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of( + new Object[]{946684800000L, "4", 6L}, + new Object[]{946684800000L, "8", 4L}, + new Object[]{946684800000L, "2", 3L}, + new Object[]{946684800000L, "3", 3L}, + new Object[]{946684800000L, "6", 3L}, + new Object[]{946684800000L, "1", 1L} + ) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + + @Test + public void testTopNOnArraysStrings() + { + final TopNQuery query = + (TopNQuery) new TopNQueryBuilder().dataSource(ARRAY) + .granularity(Granularities.ALL) + .intervals(Intervals.ONLY_ETERNITY) + .dimension(DefaultDimensionSpec.of("as")) + .metric("sum_n") + .threshold(1000) + .aggregators(new LongSumAggregatorFactory("sum_n", "n")) + .build() + .withId(UUID.randomUUID().toString()); + + + // group by cannot handle true array types, expect this, RuntimeExeception with IAE in stack trace + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Cannot create query type helper from invalid type [STRING_ARRAY]"); + + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of() + ); + } + + @Test + public void testTopNOnArraysUnknownStrings() + { + final TopNQuery query = + (TopNQuery) new TopNQueryBuilder().dataSource(ARRAY_UNKNOWN) + .granularity(Granularities.ALL) + .intervals(Intervals.ONLY_ETERNITY) + .dimension(DefaultDimensionSpec.of("as")) + .metric("sum_n") + .threshold(1000) + .aggregators(new LongSumAggregatorFactory("sum_n", "n")) + .build() + .withId(UUID.randomUUID().toString()); + + + // 'unknown' is treated as ValueType.STRING. this might not always be the case, so this is a test case of wacky + // behavior of sorts + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of( + new Object[]{946684800000L, "4.0", 6L}, + new Object[]{946684800000L, "8.0", 4L}, + new Object[]{946684800000L, "2.0", 3L}, + new Object[]{946684800000L, "3.0", 3L}, + new Object[]{946684800000L, "6.0", 3L}, + new Object[]{946684800000L, "1.0", 1L} + ) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + /** * Initialize (or reinitialize) our {@link #walker} and {@link #closer} with default scheduler. */ @@ -719,12 +1114,14 @@ public QueryRunner getQueryRunnerForSegments(Query query, Iterable>builder() + .put(FOO, makeTimeline(FOO, FOO_INLINE)) + .put(BAR, makeTimeline(BAR, BAR_INLINE)) + .put(MULTI, makeTimeline(MULTI, MULTI_VALUE_INLINE)) + .put(GLOBAL, makeTimeline(GLOBAL, FOO_INLINE)) + .put(ARRAY, makeTimeline(ARRAY, ARRAY_INLINE)) + .put(ARRAY_UNKNOWN, makeTimeline(ARRAY_UNKNOWN, ARRAY_INLINE_UNKNOWN)) + .build(), joinableFactory, conglomerate, schedulerForTest From 16882b5d8a0cadf19a60e54f4622f14cd1bf05f9 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 25 Aug 2020 18:42:03 -0700 Subject: [PATCH 22/22] timeseries longsum array --- .../server/ClientQuerySegmentWalkerTest.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java index 89364c1ed7dd..377a59fe8f7e 100644 --- a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.druid.client.DirectDruidClient; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -1030,6 +1031,58 @@ public void testTopNOnArraysUnknownStrings() Assert.assertEquals(1, scheduler.getTotalReleased().get()); } + @Test + public void testTimeseriesOnArrays() + { + final TimeseriesQuery query = + (TimeseriesQuery) Druids.newTimeseriesQueryBuilder() + .dataSource(ARRAY) + .granularity(Granularities.ALL) + .intervals(Collections.singletonList(INTERVAL)) + .aggregators(new LongSumAggregatorFactory("sum", "al")) + .context(ImmutableMap.of(TimeseriesQuery.CTX_GRAND_TOTAL, false)) + .build() + .withId(UUID.randomUUID().toString()); + + // sum doesn't know what to do with an array, so gets 0 + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of(new Object[]{INTERVAL.getStartMillis(), NullHandling.sqlCompatible() ? null : 0L}) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + + @Test + public void testTimeseriesOnArraysUnknown() + { + final TimeseriesQuery query = + (TimeseriesQuery) Druids.newTimeseriesQueryBuilder() + .dataSource(ARRAY_UNKNOWN) + .granularity(Granularities.ALL) + .intervals(Collections.singletonList(INTERVAL)) + .aggregators(new LongSumAggregatorFactory("sum", "al")) + .context(ImmutableMap.of(TimeseriesQuery.CTX_GRAND_TOTAL, false)) + .build() + .withId(UUID.randomUUID().toString()); + + // sum doesn't know what to do with an array also if type is null, so gets 0 + testQuery( + query, + ImmutableList.of(ExpectedQuery.cluster(query)), + ImmutableList.of(new Object[]{INTERVAL.getStartMillis(), NullHandling.sqlCompatible() ? null : 0L}) + ); + + Assert.assertEquals(1, scheduler.getTotalRun().get()); + Assert.assertEquals(1, scheduler.getTotalPrioritizedAndLaned().get()); + Assert.assertEquals(1, scheduler.getTotalAcquired().get()); + Assert.assertEquals(1, scheduler.getTotalReleased().get()); + } + /** * Initialize (or reinitialize) our {@link #walker} and {@link #closer} with default scheduler. */