From cd96cdc70d4da5b7b8e1b2f6ef9d692420f4c3c5 Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 24 Dec 2020 19:51:19 +0800 Subject: [PATCH 01/19] add double/long/float first/last support --- .../first/DoubleFirstAggregatorFactory.java | 3 +- .../first/FloatFirstAggregatorFactory.java | 3 +- .../first/GenericFirstAggregateCombiner.java | 71 ++++++++++++++++++ .../first/LongFirstAggregatorFactory.java | 3 +- .../last/DoubleLastAggregatorFactory.java | 2 +- .../last/FloatLastAggregatorFactory.java | 3 +- .../last/GenericLastAggregateCombiner.java | 72 +++++++++++++++++++ .../last/LongLastAggregatorFactory.java | 3 +- .../first/DoubleFirstAggregationTest.java | 35 +++++++++ .../first/FloatFirstAggregationTest.java | 35 +++++++++ .../first/LongFirstAggregationTest.java | 35 +++++++++ .../last/DoubleLastAggregationTest.java | 35 +++++++++ .../last/FloatLastAggregationTest.java | 35 +++++++++ .../last/LongLastAggregationTest.java | 35 +++++++++ 14 files changed, 359 insertions(+), 11 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 15463cf1d938..66f76708be41 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -150,7 +149,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("DoubleFirstAggregatorFactory is not supported during ingestion for rollup"); + return new GenericFirstAggregateCombiner>(){}; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index ab9211f1277c..622b4830e51d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -148,7 +147,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("FloatFirstAggregatorFactory is not supported during ingestion for rollup"); + return new GenericFirstAggregateCombiner>(){}; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java new file mode 100644 index 000000000000..ed1594124e42 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.first; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.ObjectAggregateCombiner; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; +import java.lang.reflect.ParameterizedType; + +/** + * This class is marked as abstract to prevent usage of direct type instantiation. + * To use it, just create an instance as below + * + * AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){ + * }; + * + */ +abstract public class GenericFirstAggregateCombiner> extends ObjectAggregateCombiner +{ + private T firstValue; + + @Override + public final void reset(ColumnValueSelector selector) + { + firstValue = (T) selector.getObject(); + } + + @Override + public final void fold(ColumnValueSelector selector) + { + T newValue = (T) selector.getObject(); + + if (Longs.compare(((SerializablePair) firstValue).lhs, ((SerializablePair) newValue).lhs) > 0) { + firstValue = newValue; + } + } + + @Nullable + @Override + public final T getObject() + { + return firstValue; + } + + @Override + public final Class classOfObject() + { + ParameterizedType parameterizedType = (ParameterizedType) this.getClass().getGenericSuperclass(); + return (Class) parameterizedType.getActualTypeArguments()[0]; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 697663c38574..e3f5a2bc7bf4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -147,7 +146,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("LongFirstAggregatorFactory is not supported during ingestion for rollup"); + return new GenericFirstAggregateCombiner>(){}; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 11a7e8d4b961..7704246297cd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -148,7 +148,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("DoubleLastAggregatorFactory is not supported during ingestion for rollup"); + return new GenericLastAggregateCombiner>(){}; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 01d7808c830b..d1c896649404 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -146,7 +145,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("FloatLastAggregatorFactory is not supported during ingestion for rollup"); + return new GenericLastAggregateCombiner>(){}; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java new file mode 100644 index 000000000000..50d5fdaf8edc --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.last; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.ObjectAggregateCombiner; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; +import java.lang.reflect.ParameterizedType; + +/** + * This class is marked as abstract to prevent instantiation of this class + * because it relyes on type reflection to get the right class object of generic type + * + * To use it, just create an instance as below + * AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){}; + * + */ +abstract public class GenericLastAggregateCombiner> + extends ObjectAggregateCombiner +{ + private T lastValue; + + @Override + public final void reset(ColumnValueSelector selector) + { + lastValue = (T) selector.getObject(); + } + + @Override + public final void fold(ColumnValueSelector selector) + { + T newValue = (T) selector.getObject(); + + if (Longs.compare(((SerializablePair) lastValue).lhs, ((SerializablePair) newValue).lhs) < 0) { + lastValue = newValue; + } + } + + @Nullable + @Override + public final T getObject() + { + return lastValue; + } + + @Override + public final Class classOfObject() + { + ParameterizedType parameterizedType = (ParameterizedType) this.getClass().getGenericSuperclass(); + return (Class) parameterizedType.getActualTypeArguments()[0]; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 5a8964dc3da2..8ec9405772c1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; @@ -145,7 +144,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - throw new UOE("LongLastAggregatorFactory is not supported during ingestion for rollup"); + return new GenericLastAggregateCombiner>(){}; } @Override diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java index c85b6b11c779..8b101c1ce9ba 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -22,6 +22,7 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; @@ -196,6 +197,40 @@ public void testSerde() throws Exception Assert.assertEquals(doubleFirstAggFactory, mapper.readValue(doubleSpecJson, AggregatorFactory.class)); } + @Test + public void testDoubleFirstAggregateCombiner() + { + AggregateCombiner doubleFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + + SerializablePair[] inputPairs = { + new SerializablePair<>(5L, 134.3d), + new SerializablePair<>(4L, 1232.212d), + new SerializablePair<>(3L, 18d), + new SerializablePair<>(6L, 233.5232d) + }; + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); + doubleFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[0], doubleFirstAggregateCombiner.getObject()); + + // inputPairs[1] has lower time value, it should be the first + columnSelector.increment(); + doubleFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[1], doubleFirstAggregateCombiner.getObject()); + + // inputPairs[2] has lower time value, it should be the first + columnSelector.increment(); + doubleFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], doubleFirstAggregateCombiner.getObject()); + + // inputPairs[3] has the max time value, it should NOT be the first + columnSelector.increment(); + doubleFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], doubleFirstAggregateCombiner.getObject()); + + doubleFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[3], doubleFirstAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java index b9b37f83f8f7..f3675cbb4b6c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -22,6 +22,7 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; @@ -183,6 +184,40 @@ public void testSerde() throws Exception Assert.assertEquals(floatFirstAggregatorFactory, mapper.readValue(doubleSpecJson, AggregatorFactory.class)); } + @Test + public void testFloatFirstAggregateCombiner() + { + AggregateCombiner floatFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + + SerializablePair[] inputPairs = { + new SerializablePair<>(5L, 134.3f), + new SerializablePair<>(4L, 1232.212f), + new SerializablePair<>(3L, 18f), + new SerializablePair<>(6L, 233.5232f) + }; + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); + floatFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[0], floatFirstAggregateCombiner.getObject()); + + // inputPairs[1] has lower time value, it should be the first + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[1], floatFirstAggregateCombiner.getObject()); + + // inputPairs[2] has lower time value, it should be the first + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], floatFirstAggregateCombiner.getObject()); + + // inputPairs[3] has the max time value, it should NOT be the first + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], floatFirstAggregateCombiner.getObject()); + + floatFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[3], floatFirstAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java index 7fe925656e2b..d2f5f914b4de 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -22,6 +22,7 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; @@ -182,6 +183,40 @@ public void testSerde() throws Exception Assert.assertEquals(longFirstAggFactory, mapper.readValue(longSpecJson, AggregatorFactory.class)); } + @Test + public void testLongFirstAggregateCombiner() + { + AggregateCombiner longFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + + SerializablePair[] inputPairs = { + new SerializablePair<>(5L, 134L), + new SerializablePair<>(4L, 1232L), + new SerializablePair<>(3L, 18L), + new SerializablePair<>(6L, 233L) + }; + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); + longFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[0], longFirstAggregateCombiner.getObject()); + + // inputPairs[1] has lower time value, it should be the first + columnSelector.increment(); + longFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[1], longFirstAggregateCombiner.getObject()); + + // inputPairs[2] has lower time value, it should be the first + columnSelector.increment(); + longFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], longFirstAggregateCombiner.getObject()); + + // inputPairs[3] has the max time value, it should NOT be the first + columnSelector.increment(); + longFirstAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], longFirstAggregateCombiner.getObject()); + + longFirstAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[3], longFirstAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java index 847194f12760..3d514ea1131f 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -22,6 +22,7 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; @@ -183,6 +184,40 @@ public void testSerde() throws Exception Assert.assertEquals(doubleLastAggFactory, mapper.readValue(doubleSpecJson, AggregatorFactory.class)); } + @Test + public void testDoubleLastAggregateCombiner() + { + AggregateCombiner doubleLastAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + + SerializablePair[] inputPairs = { + new SerializablePair<>(3L, 18d), + new SerializablePair<>(5L, 134.3d), + new SerializablePair<>(6L, 1232.212d), + new SerializablePair<>(1L, 233.5232d) + }; + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); + doubleLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[0], doubleLastAggregateCombiner.getObject()); + + // inputPairs[1] has larger time value, it should be the last + columnSelector.increment(); + doubleLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[1], doubleLastAggregateCombiner.getObject()); + + // inputPairs[2] has larger time value, it should be the last + columnSelector.increment(); + doubleLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], doubleLastAggregateCombiner.getObject()); + + // inputPairs[3] has the min time value, it should NOT be the first + columnSelector.increment(); + doubleLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], doubleLastAggregateCombiner.getObject()); + + doubleLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[3], doubleLastAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java index 86b6a998b8e3..976d1143c39a 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -22,6 +22,7 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; @@ -183,6 +184,40 @@ public void testSerde() throws Exception Assert.assertEquals(floatLastAggregatorFactory, mapper.readValue(doubleSpecJson, AggregatorFactory.class)); } + @Test + public void testDoubleLastAggregateCombiner() + { + AggregateCombiner floatLastAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + + SerializablePair[] inputPairs = { + new SerializablePair<>(3L, 18f), + new SerializablePair<>(5L, 134.3f), + new SerializablePair<>(6L, 1232.212f), + new SerializablePair<>(1L, 233.5232f) + }; + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); + floatLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[0], floatLastAggregateCombiner.getObject()); + + // inputPairs[1] has larger time value, it should be the last + columnSelector.increment(); + floatLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[1], floatLastAggregateCombiner.getObject()); + + // inputPairs[2] has larger time value, it should be the last + columnSelector.increment(); + floatLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], floatLastAggregateCombiner.getObject()); + + // inputPairs[3] has the min time value, it should NOT be the first + columnSelector.increment(); + floatLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], floatLastAggregateCombiner.getObject()); + + floatLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[3], floatLastAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java index a9b2fadcc90e..d4d604108e91 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java @@ -22,6 +22,7 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.BufferAggregator; @@ -182,6 +183,40 @@ public void testSerde() throws Exception Assert.assertEquals(longLastAggFactory, mapper.readValue(longSpecJson, AggregatorFactory.class)); } + @Test + public void testLongLastAggregateCombiner() + { + AggregateCombiner longLastAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); + + SerializablePair[] inputPairs = { + new SerializablePair<>(3L, 18L), + new SerializablePair<>(5L, 134L), + new SerializablePair<>(6L, 1232L), + new SerializablePair<>(1L, 2332L) + }; + TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); + longLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[0], longLastAggregateCombiner.getObject()); + + // inputPairs[1] has larger time value, it should be the last + columnSelector.increment(); + longLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[1], longLastAggregateCombiner.getObject()); + + // inputPairs[2] has larger time value, it should be the last + columnSelector.increment(); + longLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], longLastAggregateCombiner.getObject()); + + // inputPairs[3] has the min time value, it should NOT be the first + columnSelector.increment(); + longLastAggregateCombiner.fold(columnSelector); + Assert.assertEquals(inputPairs[2], longLastAggregateCombiner.getObject()); + + longLastAggregateCombiner.reset(columnSelector); + Assert.assertEquals(inputPairs[3], longLastAggregateCombiner.getObject()); + } + private void aggregate( Aggregator agg ) From a2b843534b65718e56118a328b187ae39867e804 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 25 Dec 2020 15:33:15 +0800 Subject: [PATCH 02/19] fix index rollup --- .../druid/jackson/AggregatorsModule.java | 8 +- .../SerializablePairLongDoubleSerde.java | 128 ++++++++++++++++++ .../SerializablePairLongFloatSerde.java | 128 ++++++++++++++++++ .../SerializablePairLongLongSerde.java | 128 ++++++++++++++++++ .../SerializablePairLongStringSerde.java | 2 +- .../first/DoubleFirstAggregatorFactory.java | 13 +- .../first/FloatFirstAggregatorFactory.java | 17 ++- .../first/GenericFirstAggregateCombiner.java | 8 +- .../first/LongFirstAggregatorFactory.java | 17 ++- .../last/DoubleLastAggregatorFactory.java | 14 +- .../last/FloatLastAggregatorFactory.java | 13 +- .../last/GenericLastAggregateCombiner.java | 7 +- .../last/LongLastAggregatorFactory.java | 13 +- .../druid/segment/IndexMergerRollupTest.java | 127 +++++++++++++---- 14 files changed, 572 insertions(+), 51 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java diff --git a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java index 795ea5b6d31c..5a9de6f1bf07 100644 --- a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java @@ -38,6 +38,9 @@ import org.apache.druid.query.aggregation.LongMinAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; +import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; +import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; import org.apache.druid.query.aggregation.SerializablePairLongStringSerde; import org.apache.druid.query.aggregation.any.DoubleAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.FloatAnyAggregatorFactory; @@ -80,7 +83,10 @@ public AggregatorsModule() ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde()); ComplexMetrics.registerSerde("preComputedHyperUnique", new PreComputedHyperUniquesSerde()); - ComplexMetrics.registerSerde("serializablePairLongString", new SerializablePairLongStringSerde()); + ComplexMetrics.registerSerde(SerializablePairLongStringSerde.TYPE_NAME, new SerializablePairLongStringSerde()); + ComplexMetrics.registerSerde(SerializablePairLongDoubleSerde.TYPE_NAME, new SerializablePairLongDoubleSerde()); + ComplexMetrics.registerSerde(SerializablePairLongFloatSerde.TYPE_NAME, new SerializablePairLongFloatSerde()); + ComplexMetrics.registerSerde(SerializablePairLongLongSerde.TYPE_NAME, new SerializablePairLongLongSerde()); setMixInAnnotation(AggregatorFactory.class, AggregatorFactoryMixin.class); setMixInAnnotation(PostAggregator.class, PostAggregatorMixin.class); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java new file mode 100644 index 000000000000..0ed7e803fe92 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * The class serializes a Long-Double pair (SerializablePair). + * The serialization structure is: Long:Double + *

+ * The class is used on first/last Double aggregators to store the time and the first/last Double. + * Long:Long -> Timestamp:Long + */ +public class SerializablePairLongDoubleSerde extends ComplexMetricSerde +{ + public static final String TYPE_NAME = "serializablePairLongDouble"; + + @Override + public String getTypeName() + { + return TYPE_NAME; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + SerializablePair pair = new SerializablePair<>(null, null); + final Class> pairClass = (Class>) pair.getClass(); + return new ComplexMetricExtractor() + { + @Override + public Class> extractedClass() + { + return pairClass; + } + + @Override + public Object extractValue(InputRow inputRow, String metricName) + { + return inputRow.getRaw(metricName); + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public ObjectStrategy getObjectStrategy() + { + SerializablePair pair = new SerializablePair<>(null, null); + final Class> pairClass = (Class>) pair.getClass(); + + return new ObjectStrategy>() + { + @Override + public int compare(@Nullable SerializablePair o1, @Nullable SerializablePair o2) + { + return Longs.compare(o1.lhs, o2.lhs); + } + + @Override + public Class> getClazz() + { + return pairClass; + } + + @Override + public SerializablePair fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + double rhs = readOnlyBuffer.getDouble(); + return new SerializablePair(lhs, rhs); + } + + @Override + public byte[] toBytes(SerializablePair val) + { + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Double.BYTES); + bbuf.putLong(val.lhs); + bbuf.putDouble(val.rhs); + return bbuf.array(); + } + }; + } + + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java new file mode 100644 index 000000000000..a639f26f57c1 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * The class serializes a Long-Float pair (SerializablePair). + * The serialization structure is: Long:Float + *

+ * The class is used on first/last Float aggregators to store the time and the first/last Float. + * Long:Long -> Timestamp:Long + */ +public class SerializablePairLongFloatSerde extends ComplexMetricSerde +{ + public static final String TYPE_NAME = "serializablePairLongFloat"; + + @Override + public String getTypeName() + { + return TYPE_NAME; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + SerializablePair pair = new SerializablePair<>(null, null); + final Class> pairClass = (Class>) pair.getClass(); + return new ComplexMetricExtractor() + { + @Override + public Class> extractedClass() + { + return pairClass; + } + + @Override + public Object extractValue(InputRow inputRow, String metricName) + { + return inputRow.getRaw(metricName); + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public ObjectStrategy getObjectStrategy() + { + SerializablePair pair = new SerializablePair<>(null, null); + final Class> pairClass = (Class>) pair.getClass(); + + return new ObjectStrategy>() + { + @Override + public int compare(@Nullable SerializablePair o1, @Nullable SerializablePair o2) + { + return Longs.compare(o1.lhs, o2.lhs); + } + + @Override + public Class> getClazz() + { + return pairClass; + } + + @Override + public SerializablePair fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + float rhs = readOnlyBuffer.getFloat(); + return new SerializablePair(lhs, rhs); + } + + @Override + public byte[] toBytes(SerializablePair val) + { + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Float.BYTES); + bbuf.putLong(val.lhs); + bbuf.putFloat(val.rhs); + return bbuf.array(); + } + }; + } + + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java new file mode 100644 index 000000000000..29bf3ef40e14 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * The class serializes a Long-Long pair (SerializablePair). + * The serialization structure is: Long:Long + *

+ * The class is used on first/last Long aggregators to store the time and the first/last Long. + * Long:Long -> Timestamp:Long + */ +public class SerializablePairLongLongSerde extends ComplexMetricSerde +{ + public static final String TYPE_NAME = "serializablePairLongLong"; + + @Override + public String getTypeName() + { + return TYPE_NAME; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + SerializablePair pair = new SerializablePair<>(null, null); + final Class> pairClass = (Class>) pair.getClass(); + return new ComplexMetricExtractor() + { + @Override + public Class> extractedClass() + { + return pairClass; + } + + @Override + public Object extractValue(InputRow inputRow, String metricName) + { + return inputRow.getRaw(metricName); + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public ObjectStrategy getObjectStrategy() + { + SerializablePair pair = new SerializablePair<>(null, null); + final Class> pairClass = (Class>) pair.getClass(); + + return new ObjectStrategy>() + { + @Override + public int compare(@Nullable SerializablePair o1, @Nullable SerializablePair o2) + { + return Longs.compare(o1.lhs, o2.lhs); + } + + @Override + public Class> getClazz() + { + return pairClass; + } + + @Override + public SerializablePair fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + long rhs = readOnlyBuffer.getLong(); + return new SerializablePair(lhs, rhs); + } + + @Override + public byte[] toBytes(SerializablePair val) + { + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Long.BYTES); + bbuf.putLong(val.lhs); + bbuf.putLong(val.rhs); + return bbuf.array(); + } + }; + } + + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSerde.java index ab40ec7a1899..41a08ff43d03 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongStringSerde.java @@ -45,7 +45,7 @@ public class SerializablePairLongStringSerde extends ComplexMetricSerde { - private static final String TYPE_NAME = "serializablePairLongString"; + public static final String TYPE_NAME = "serializablePairLongString"; @Override public String getTypeName() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 66f76708be41..deb7b52ec277 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -272,11 +273,19 @@ public byte[] getCacheKey() .array(); } + @Override + public String getComplexTypeName() + { + return SerializablePairLongDoubleSerde.TYPE_NAME; + } + + /** + * actual type is {@link SerializablePair} + */ @Override public ValueType getType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; + return ValueType.COMPLEX; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 622b4830e51d..3a6083a275fa 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -147,7 +148,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericFirstAggregateCombiner>(){}; + return new GenericFirstAggregateCombiner>() + { + }; } @Override @@ -269,11 +272,19 @@ public byte[] getCacheKey() .array(); } + @Override + public String getComplexTypeName() + { + return SerializablePairLongFloatSerde.TYPE_NAME; + } + + /** + * actual type is {@link SerializablePair} + */ @Override public ValueType getType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.FLOAT; + return ValueType.COMPLEX; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java index ed1594124e42..b334ddd8fe9b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java @@ -28,14 +28,16 @@ import java.lang.reflect.ParameterizedType; /** - * This class is marked as abstract to prevent usage of direct type instantiation. - * To use it, just create an instance as below + * This class is used for FIRST aggregator + * + * It's marked as abstract to prevent instantiation of this type directly. + * it must be used it as follow: * * AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){ * }; * */ -abstract public class GenericFirstAggregateCombiner> extends ObjectAggregateCombiner +public abstract class GenericFirstAggregateCombiner> extends ObjectAggregateCombiner { private T firstValue; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index e3f5a2bc7bf4..486940fd48f1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; @@ -146,7 +147,9 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericFirstAggregateCombiner>(){}; + return new GenericFirstAggregateCombiner>() + { + }; } @Override @@ -267,11 +270,19 @@ public byte[] getCacheKey() .array(); } + @Override + public String getComplexTypeName() + { + return SerializablePairLongLongSerde.TYPE_NAME; + } + + /** + * actual type is {@link SerializablePair} + */ @Override public ValueType getType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.LONG; + return ValueType.COMPLEX; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 7704246297cd..f277897643c0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -24,12 +24,12 @@ import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.AggregateCombiner; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -271,11 +271,19 @@ public byte[] getCacheKey() .array(); } + @Override + public String getComplexTypeName() + { + return SerializablePairLongDoubleSerde.TYPE_NAME; + } + + /** + * actual type is {@link SerializablePair} + */ @Override public ValueType getType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return storeDoubleAsFloat ? ValueType.FLOAT : ValueType.DOUBLE; + return ValueType.COMPLEX; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index d1c896649404..e1988b405d9b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -267,11 +268,19 @@ public byte[] getCacheKey() .array(); } + @Override + public String getComplexTypeName() + { + return SerializablePairLongFloatSerde.TYPE_NAME; + } + + /** + * actual type is {@link SerializablePair} + */ @Override public ValueType getType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.FLOAT; + return ValueType.COMPLEX; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java index 50d5fdaf8edc..535c388d0d95 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -28,15 +28,14 @@ import java.lang.reflect.ParameterizedType; /** - * This class is marked as abstract to prevent instantiation of this class + * This class is marked as abstract to prevent instantiation of this type directly, * because it relyes on type reflection to get the right class object of generic type * - * To use it, just create an instance as below + * use it as follow: * AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){}; * */ -abstract public class GenericLastAggregateCombiner> - extends ObjectAggregateCombiner +public abstract class GenericLastAggregateCombiner> extends ObjectAggregateCombiner { private T lastValue; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 8ec9405772c1..f9df6b349b5f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -265,11 +266,19 @@ public byte[] getCacheKey() .array(); } + @Override + public String getComplexTypeName() + { + return SerializablePairLongLongSerde.TYPE_NAME; + } + + /** + * actual type is {@link SerializablePair} + */ @Override public ValueType getType() { - // if we don't pretend to be a primitive, group by v1 gets sad and doesn't work because no complex type serde - return ValueType.LONG; + return ValueType.COMPLEX; } @Override diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index 3cd0e8620ced..c937ab7f664d 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -20,9 +20,16 @@ package org.apache.druid.segment; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; import org.apache.druid.segment.data.IncrementalIndexTest; import org.apache.druid.segment.incremental.IncrementalIndex; @@ -38,7 +45,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,6 +58,26 @@ public class IndexMergerRollupTest extends InitializedNullHandlingTest @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private final List> strEventsList = Arrays.asList( + ImmutableMap.of("d", "d1", "m", "m1"), + ImmutableMap.of("d", "d1", "m", "m2") + ); + + private final List> doubleEventsList = Arrays.asList( + ImmutableMap.of("d", "d1", "m", 1.0d), + ImmutableMap.of("d", "d1", "m", 2.0d) + ); + + private final List> floatEventsList = Arrays.asList( + ImmutableMap.of("d", "d1", "m", 1.0f), + ImmutableMap.of("d", "d1", "m", 2.0f) + ); + + private final List> longEventsList = Arrays.asList( + ImmutableMap.of("d", "d1", "m", 1L), + ImmutableMap.of("d", "d1", "m", 2L) + ); + @Before public void setUp() { @@ -61,27 +87,12 @@ public void setUp() indexSpec = new IndexSpec(); } - private void testStringFirstLastRollup( - AggregatorFactory[] aggregatorFactories + private void testFirstLastRollup( + final List> eventsList, + final List dimensions, + final AggregatorFactory... aggregatorFactories ) throws Exception { - List> eventsList = Arrays.asList( - new HashMap() - { - { - put("d", "d1"); - put("m", "m1"); - } - }, - new HashMap() - { - { - put("d", "d1"); - put("m", "m2"); - } - } - ); - final File tempDir = temporaryFolder.newFolder(); List indexes = new ArrayList<>(); @@ -90,7 +101,7 @@ private void testStringFirstLastRollup( for (Map events : eventsList) { IncrementalIndex toPersist = IncrementalIndexTest.createIndex(aggregatorFactories); - toPersist.add(new MapBasedInputRow(time.toEpochMilli(), ImmutableList.of("d"), events)); + toPersist.add(new MapBasedInputRow(time.toEpochMilli(), dimensions, events)); indexes.add(indexIO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec, null))); } @@ -104,18 +115,80 @@ private void testStringFirstLastRollup( @Test public void testStringFirstRollup() throws Exception { - AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{ + testFirstLastRollup( + strEventsList, + ImmutableList.of("d"), new StringFirstAggregatorFactory("m", "m", 1024) - }; - testStringFirstLastRollup(aggregatorFactories); + ); } @Test public void testStringLastRollup() throws Exception { - AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{ + testFirstLastRollup( + strEventsList, + ImmutableList.of("d"), new StringLastAggregatorFactory("m", "m", 1024) - }; - testStringFirstLastRollup(aggregatorFactories); + ); + } + + @Test + public void testDoubleFirstRollup() throws Exception + { + testFirstLastRollup( + doubleEventsList, + ImmutableList.of("d"), + new DoubleFirstAggregatorFactory("m", "m") + ); + } + + @Test + public void testDoubleLastRollup() throws Exception + { + testFirstLastRollup( + doubleEventsList, + ImmutableList.of("d"), + new DoubleLastAggregatorFactory("m", "m") + ); + } + + @Test + public void testFloatFirstRollup() throws Exception + { + testFirstLastRollup( + doubleEventsList, + ImmutableList.of("d"), + new FloatFirstAggregatorFactory("m", "m") + ); + } + + @Test + public void testFloatLastRollup() throws Exception + { + testFirstLastRollup( + doubleEventsList, + ImmutableList.of("d"), + new FloatLastAggregatorFactory("m", "m") + ); + } + + @Test + public void testLongFirstRollup() throws Exception + { + testFirstLastRollup( + doubleEventsList, + ImmutableList.of("d"), + new LongFirstAggregatorFactory("m", "m") + ); + } + + @Test + public void testLongLastRollup() throws Exception + { + testFirstLastRollup( + doubleEventsList, + ImmutableList.of("d"), + new LongLastAggregatorFactory("m", "m") + ); } } From fe84826a580a817681c94f78f7047d8a54054c68 Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 28 Jan 2021 20:41:14 +0800 Subject: [PATCH 03/19] fix type --- .../druid/jackson/AggregatorsModule.java | 8 +- .../AbstractSerializablePairSerde.java | 118 ++++++++++++++++++ .../SerializablePairLongDouble.java | 37 ++++++ .../SerializablePairLongDoubleSerde.java | 95 +++----------- .../SerializablePairLongFloat.java | 37 ++++++ .../SerializablePairLongFloatSerde.java | 94 +++----------- .../aggregation/SerializablePairLongLong.java | 37 ++++++ .../SerializablePairLongLongSerde.java | 94 +++----------- .../first/DoubleFirstAggregator.java | 4 +- .../first/DoubleFirstAggregatorFactory.java | 3 +- .../first/FloatFirstAggregator.java | 4 +- .../first/FloatFirstAggregatorFactory.java | 5 +- .../first/GenericFirstAggregateCombiner.java | 20 +-- .../first/LongFirstAggregator.java | 4 +- .../first/LongFirstAggregatorFactory.java | 5 +- .../last/DoubleLastAggregator.java | 4 +- .../last/DoubleLastAggregatorFactory.java | 3 +- .../aggregation/last/FloatLastAggregator.java | 3 +- .../last/FloatLastAggregatorFactory.java | 3 +- .../last/GenericLastAggregateCombiner.java | 21 ++-- .../aggregation/last/LongLastAggregator.java | 4 +- .../last/LongLastAggregatorFactory.java | 3 +- .../druid/segment/serde/ComplexMetrics.java | 8 ++ .../druid/segment/IndexMergerRollupTest.java | 8 +- 24 files changed, 339 insertions(+), 283 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java diff --git a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java index 5a9de6f1bf07..0d3e2dea8efc 100644 --- a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java @@ -83,10 +83,10 @@ public AggregatorsModule() ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde()); ComplexMetrics.registerSerde("preComputedHyperUnique", new PreComputedHyperUniquesSerde()); - ComplexMetrics.registerSerde(SerializablePairLongStringSerde.TYPE_NAME, new SerializablePairLongStringSerde()); - ComplexMetrics.registerSerde(SerializablePairLongDoubleSerde.TYPE_NAME, new SerializablePairLongDoubleSerde()); - ComplexMetrics.registerSerde(SerializablePairLongFloatSerde.TYPE_NAME, new SerializablePairLongFloatSerde()); - ComplexMetrics.registerSerde(SerializablePairLongLongSerde.TYPE_NAME, new SerializablePairLongLongSerde()); + ComplexMetrics.registerSerde(new SerializablePairLongStringSerde()); + ComplexMetrics.registerSerde(new SerializablePairLongDoubleSerde()); + ComplexMetrics.registerSerde(new SerializablePairLongFloatSerde()); + ComplexMetrics.registerSerde(new SerializablePairLongLongSerde()); setMixInAnnotation(AggregatorFactory.class, AggregatorFactoryMixin.class); setMixInAnnotation(PostAggregator.class, PostAggregatorMixin.class); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java new file mode 100644 index 000000000000..dccb389adede --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.primitives.Longs; +import org.apache.druid.collections.SerializablePair; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.segment.GenericColumnSerializer; +import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.data.GenericIndexed; +import org.apache.druid.segment.data.ObjectStrategy; +import org.apache.druid.segment.serde.ComplexColumnPartSupplier; +import org.apache.druid.segment.serde.ComplexMetricExtractor; +import org.apache.druid.segment.serde.ComplexMetricSerde; +import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; +import org.apache.druid.segment.writeout.SegmentWriteOutMedium; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * The class serializes a Long-X pair object + *

+ * Note: T must be a concrete class which inherits from SerializablePair, such as {@link SerializablePairLongDouble} + */ +public abstract class AbstractSerializablePairSerde> extends ComplexMetricSerde +{ + private final Class pairClassObject; + + public AbstractSerializablePairSerde(Class pairClassObject) + { + this.pairClassObject = pairClassObject; + } + + @Override + public ComplexMetricExtractor getExtractor() + { + return new ComplexMetricExtractor() + { + @Override + public Class extractedClass() + { + return pairClassObject; + } + + @Override + public T extractValue(InputRow inputRow, String metricName) + { + return (T)inputRow.getRaw(metricName); + } + }; + } + + @Override + public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + { + final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); + columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + } + + @Override + public ObjectStrategy getObjectStrategy() + { + return new ObjectStrategy() + { + @Override + public int compare(@Nullable T o1, @Nullable T o2) + { + return Longs.compare(o1.lhs, o2.lhs); + } + + @Override + public Class getClazz() + { + return pairClassObject; + } + + @Override + public T fromByteBuffer(ByteBuffer buffer, int numBytes) + { + return toPairObject(buffer, numBytes); + } + + @Override + public byte[] toBytes(T val) + { + return pairToBytes(val); + } + }; + } + + @Override + public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + { + return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + } + + abstract protected T toPairObject(ByteBuffer buffer, int numBytes); + + abstract protected byte[] pairToBytes(T val); +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java new file mode 100644 index 000000000000..199a214ac77a --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDouble.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.collections.SerializablePair; + +import javax.annotation.Nullable; + +public class SerializablePairLongDouble extends SerializablePair +{ + @JsonCreator + public SerializablePairLongDouble(@JsonProperty("lhs") Long lhs, @JsonProperty("rhs") @Nullable Double rhs) + { + super(lhs, rhs); + } +} + + diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 0ed7e803fe92..1910bad74d86 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -19,20 +19,6 @@ package org.apache.druid.query.aggregation; -import com.google.common.primitives.Longs; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.segment.GenericColumnSerializer; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.data.GenericIndexed; -import org.apache.druid.segment.data.ObjectStrategy; -import org.apache.druid.segment.serde.ComplexColumnPartSupplier; -import org.apache.druid.segment.serde.ComplexMetricExtractor; -import org.apache.druid.segment.serde.ComplexMetricSerde; -import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; -import org.apache.druid.segment.writeout.SegmentWriteOutMedium; - -import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -42,87 +28,36 @@ * The class is used on first/last Double aggregators to store the time and the first/last Double. * Long:Long -> Timestamp:Long */ -public class SerializablePairLongDoubleSerde extends ComplexMetricSerde +public class SerializablePairLongDoubleSerde extends AbstractSerializablePairSerde { public static final String TYPE_NAME = "serializablePairLongDouble"; - @Override - public String getTypeName() + public SerializablePairLongDoubleSerde() { - return TYPE_NAME; + super(SerializablePairLongDouble.class); } @Override - public ComplexMetricExtractor getExtractor() - { - SerializablePair pair = new SerializablePair<>(null, null); - final Class> pairClass = (Class>) pair.getClass(); - return new ComplexMetricExtractor() - { - @Override - public Class> extractedClass() - { - return pairClass; - } - - @Override - public Object extractValue(InputRow inputRow, String metricName) - { - return inputRow.getRaw(metricName); - } - }; - } - - @Override - public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + public String getTypeName() { - final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + return TYPE_NAME; } @Override - public ObjectStrategy getObjectStrategy() + protected SerializablePairLongDouble toPairObject(ByteBuffer buffer, int numBytes) { - SerializablePair pair = new SerializablePair<>(null, null); - final Class> pairClass = (Class>) pair.getClass(); - - return new ObjectStrategy>() - { - @Override - public int compare(@Nullable SerializablePair o1, @Nullable SerializablePair o2) - { - return Longs.compare(o1.lhs, o2.lhs); - } - - @Override - public Class> getClazz() - { - return pairClass; - } - - @Override - public SerializablePair fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - double rhs = readOnlyBuffer.getDouble(); - return new SerializablePair(lhs, rhs); - } - - @Override - public byte[] toBytes(SerializablePair val) - { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Double.BYTES); - bbuf.putLong(val.lhs); - bbuf.putDouble(val.rhs); - return bbuf.array(); - } - }; + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + double rhs = readOnlyBuffer.getDouble(); + return new SerializablePairLongDouble(lhs, rhs); } @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) + protected byte[] pairToBytes(SerializablePairLongDouble val) { - return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Double.BYTES); + bbuf.putLong(val.lhs); + bbuf.putDouble(val.rhs); + return bbuf.array(); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java new file mode 100644 index 000000000000..6ecc0f9533e9 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloat.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.collections.SerializablePair; + +import javax.annotation.Nullable; + +public class SerializablePairLongFloat extends SerializablePair +{ + @JsonCreator + public SerializablePairLongFloat(@JsonProperty("lhs") Long lhs, @JsonProperty("rhs") @Nullable Float rhs) + { + super(lhs, rhs); + } +} + + diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index a639f26f57c1..87f277aa4c1d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -19,20 +19,6 @@ package org.apache.druid.query.aggregation; -import com.google.common.primitives.Longs; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.segment.GenericColumnSerializer; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.data.GenericIndexed; -import org.apache.druid.segment.data.ObjectStrategy; -import org.apache.druid.segment.serde.ComplexColumnPartSupplier; -import org.apache.druid.segment.serde.ComplexMetricExtractor; -import org.apache.druid.segment.serde.ComplexMetricSerde; -import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; -import org.apache.druid.segment.writeout.SegmentWriteOutMedium; - -import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -42,87 +28,37 @@ * The class is used on first/last Float aggregators to store the time and the first/last Float. * Long:Long -> Timestamp:Long */ -public class SerializablePairLongFloatSerde extends ComplexMetricSerde +public class SerializablePairLongFloatSerde extends AbstractSerializablePairSerde { public static final String TYPE_NAME = "serializablePairLongFloat"; - @Override - public String getTypeName() + public SerializablePairLongFloatSerde() { - return TYPE_NAME; + super(SerializablePairLongFloat.class); } @Override - public ComplexMetricExtractor getExtractor() + public String getTypeName() { - SerializablePair pair = new SerializablePair<>(null, null); - final Class> pairClass = (Class>) pair.getClass(); - return new ComplexMetricExtractor() - { - @Override - public Class> extractedClass() - { - return pairClass; - } - - @Override - public Object extractValue(InputRow inputRow, String metricName) - { - return inputRow.getRaw(metricName); - } - }; + return TYPE_NAME; } @Override - public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + protected SerializablePairLongFloat toPairObject(ByteBuffer buffer, int numBytes) { - final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + float rhs = readOnlyBuffer.getFloat(); + return new SerializablePairLongFloat(lhs, rhs); } @Override - public ObjectStrategy getObjectStrategy() + protected byte[] pairToBytes(SerializablePairLongFloat val) { - SerializablePair pair = new SerializablePair<>(null, null); - final Class> pairClass = (Class>) pair.getClass(); - - return new ObjectStrategy>() - { - @Override - public int compare(@Nullable SerializablePair o1, @Nullable SerializablePair o2) - { - return Longs.compare(o1.lhs, o2.lhs); - } - - @Override - public Class> getClazz() - { - return pairClass; - } - - @Override - public SerializablePair fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - float rhs = readOnlyBuffer.getFloat(); - return new SerializablePair(lhs, rhs); - } - - @Override - public byte[] toBytes(SerializablePair val) - { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Float.BYTES); - bbuf.putLong(val.lhs); - bbuf.putFloat(val.rhs); - return bbuf.array(); - } - }; + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Float.BYTES); + bbuf.putLong(val.lhs); + bbuf.putFloat(val.rhs); + return bbuf.array(); } - @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) - { - return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java new file mode 100644 index 000000000000..82f9f191b1f6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLong.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.collections.SerializablePair; + +import javax.annotation.Nullable; + +public class SerializablePairLongLong extends SerializablePair +{ + @JsonCreator + public SerializablePairLongLong(@JsonProperty("lhs") Long lhs, @JsonProperty("rhs") @Nullable Long rhs) + { + super(lhs, rhs); + } +} + + diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index 29bf3ef40e14..cbbd7303a817 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -19,20 +19,6 @@ package org.apache.druid.query.aggregation; -import com.google.common.primitives.Longs; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.segment.GenericColumnSerializer; -import org.apache.druid.segment.column.ColumnBuilder; -import org.apache.druid.segment.data.GenericIndexed; -import org.apache.druid.segment.data.ObjectStrategy; -import org.apache.druid.segment.serde.ComplexColumnPartSupplier; -import org.apache.druid.segment.serde.ComplexMetricExtractor; -import org.apache.druid.segment.serde.ComplexMetricSerde; -import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; -import org.apache.druid.segment.writeout.SegmentWriteOutMedium; - -import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -42,87 +28,37 @@ * The class is used on first/last Long aggregators to store the time and the first/last Long. * Long:Long -> Timestamp:Long */ -public class SerializablePairLongLongSerde extends ComplexMetricSerde +public class SerializablePairLongLongSerde extends AbstractSerializablePairSerde { public static final String TYPE_NAME = "serializablePairLongLong"; - @Override - public String getTypeName() + public SerializablePairLongLongSerde() { - return TYPE_NAME; + super(SerializablePairLongLong.class); } @Override - public ComplexMetricExtractor getExtractor() + public String getTypeName() { - SerializablePair pair = new SerializablePair<>(null, null); - final Class> pairClass = (Class>) pair.getClass(); - return new ComplexMetricExtractor() - { - @Override - public Class> extractedClass() - { - return pairClass; - } - - @Override - public Object extractValue(InputRow inputRow, String metricName) - { - return inputRow.getRaw(metricName); - } - }; + return TYPE_NAME; } @Override - public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) + protected SerializablePairLongLong toPairObject(ByteBuffer buffer, int numBytes) { - final GenericIndexed column = GenericIndexed.read(buffer, getObjectStrategy(), columnBuilder.getFileMapper()); - columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + long rhs = readOnlyBuffer.getLong(); + return new SerializablePairLongLong(lhs, rhs); } @Override - public ObjectStrategy getObjectStrategy() + protected byte[] pairToBytes(SerializablePairLongLong val) { - SerializablePair pair = new SerializablePair<>(null, null); - final Class> pairClass = (Class>) pair.getClass(); - - return new ObjectStrategy>() - { - @Override - public int compare(@Nullable SerializablePair o1, @Nullable SerializablePair o2) - { - return Longs.compare(o1.lhs, o2.lhs); - } - - @Override - public Class> getClazz() - { - return pairClass; - } - - @Override - public SerializablePair fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - long rhs = readOnlyBuffer.getLong(); - return new SerializablePair(lhs, rhs); - } - - @Override - public byte[] toBytes(SerializablePair val) - { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Long.BYTES); - bbuf.putLong(val.lhs); - bbuf.putLong(val.rhs); - return bbuf.array(); - } - }; + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Long.BYTES); + bbuf.putLong(val.lhs); + bbuf.putLong(val.rhs); + return bbuf.array(); } - @Override - public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) - { - return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java index 8b4a89d0d72e..e68c7c1ada8c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -42,7 +42,7 @@ void setCurrentValue() @Override public Object get() { - return new SerializablePair<>(firstTime, rhsNull ? null : firstValue); + return new SerializablePairLongDouble(firstTime, rhsNull ? null : firstValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index deb7b52ec277..b10574b68509 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseDoubleColumnValueSelector; @@ -150,7 +151,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericFirstAggregateCombiner>(){}; + return new GenericFirstAggregateCombiner(SerializablePairLongDouble.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java index 2c0f62934ffe..a563d7f30955 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -45,7 +45,7 @@ void setCurrentValue() @Override public Object get() { - return new SerializablePair<>(firstTime, rhsNull ? null : firstValue); + return new SerializablePairLongFloat(firstTime, rhsNull ? null : firstValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 3a6083a275fa..11c2fe9ea24e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseFloatColumnValueSelector; @@ -148,9 +149,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericFirstAggregateCombiner>() - { - }; + return new GenericFirstAggregateCombiner(SerializablePairLongFloat.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java index b334ddd8fe9b..9972728274f3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java @@ -25,22 +25,27 @@ import org.apache.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; -import java.lang.reflect.ParameterizedType; /** * This class is used for FIRST aggregator - * + *

* It's marked as abstract to prevent instantiation of this type directly. * it must be used it as follow: - * + *

* AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){ * }; - * */ -public abstract class GenericFirstAggregateCombiner> extends ObjectAggregateCombiner +final public class GenericFirstAggregateCombiner> + extends ObjectAggregateCombiner { + private final Class pairClass; private T firstValue; + public GenericFirstAggregateCombiner(Class pairClass) + { + this.pairClass = pairClass; + } + @Override public final void reset(ColumnValueSelector selector) { @@ -52,7 +57,7 @@ public final void fold(ColumnValueSelector selector) { T newValue = (T) selector.getObject(); - if (Longs.compare(((SerializablePair) firstValue).lhs, ((SerializablePair) newValue).lhs) > 0) { + if (Longs.compare(firstValue.lhs, newValue.lhs) > 0) { firstValue = newValue; } } @@ -67,7 +72,6 @@ public final T getObject() @Override public final Class classOfObject() { - ParameterizedType parameterizedType = (ParameterizedType) this.getClass().getGenericSuperclass(); - return (Class) parameterizedType.getActualTypeArguments()[0]; + return this.pairClass; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java index 8cda544521ea..2fc74de7c6b8 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; public class LongFirstAggregator extends NumericFirstAggregator @@ -41,7 +41,7 @@ void setCurrentValue() @Override public Object get() { - return new SerializablePair<>(firstTime, rhsNull ? null : firstValue); + return new SerializablePairLongLong(firstTime, rhsNull ? null : firstValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 486940fd48f1..78498f5ce9e1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -147,9 +148,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericFirstAggregateCombiner>() - { - }; + return new GenericFirstAggregateCombiner(SerializablePairLongLong.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java index 3f6a1506bad6..213f8085e59e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -42,7 +42,7 @@ void setCurrentValue() @Override public Object get() { - return new SerializablePair<>(lastTime, rhsNull ? null : lastValue); + return new SerializablePairLongDouble(lastTime, rhsNull ? null : lastValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index f277897643c0..216135618089 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; @@ -148,7 +149,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericLastAggregateCombiner>(){}; + return new GenericLastAggregateCombiner(SerializablePairLongDouble.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java index 1381ccb18b7a..293c7ca4bb89 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -42,7 +43,7 @@ void setCurrentValue() @Override public Object get() { - return new SerializablePair<>(lastTime, rhsNull ? null : lastValue); + return new SerializablePairLongFloat(lastTime, rhsNull ? null : lastValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index e1988b405d9b..ad336897c7f9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; @@ -146,7 +147,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericLastAggregateCombiner>(){}; + return new GenericLastAggregateCombiner(SerializablePairLongFloat.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java index 535c388d0d95..9480dde57242 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -25,20 +25,26 @@ import org.apache.druid.segment.ColumnValueSelector; import javax.annotation.Nullable; -import java.lang.reflect.ParameterizedType; /** * This class is marked as abstract to prevent instantiation of this type directly, * because it relyes on type reflection to get the right class object of generic type - * + *

* use it as follow: - * AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){}; - * + * AggregateCombiner combiner = new GenericLastAggregateCombiner>(){}; */ -public abstract class GenericLastAggregateCombiner> extends ObjectAggregateCombiner +final public class GenericLastAggregateCombiner> + extends ObjectAggregateCombiner { private T lastValue; + private final Class pairClass; + + public GenericLastAggregateCombiner(Class pairClass) + { + this.pairClass = pairClass; + } + @Override public final void reset(ColumnValueSelector selector) { @@ -50,7 +56,7 @@ public final void fold(ColumnValueSelector selector) { T newValue = (T) selector.getObject(); - if (Longs.compare(((SerializablePair) lastValue).lhs, ((SerializablePair) newValue).lhs) < 0) { + if (Longs.compare(lastValue.lhs, newValue.lhs) < 0) { lastValue = newValue; } } @@ -65,7 +71,6 @@ public final T getObject() @Override public final Class classOfObject() { - ParameterizedType parameterizedType = (ParameterizedType) this.getClass().getGenericSuperclass(); - return (Class) parameterizedType.getActualTypeArguments()[0]; + return this.pairClass; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java index 59a159d2d875..9d9ebe0fd4ba 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java @@ -19,7 +19,7 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; public class LongLastAggregator extends NumericLastAggregator @@ -41,7 +41,7 @@ void setCurrentValue() @Override public Object get() { - return new SerializablePair<>(lastTime, rhsNull ? null : lastValue); + return new SerializablePairLongLong(lastTime, rhsNull ? null : lastValue); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index f9df6b349b5f..6ef12ecbd60a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -29,6 +29,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; @@ -145,7 +146,7 @@ public Object combine(@Nullable Object lhs, @Nullable Object rhs) @Override public AggregateCombiner makeAggregateCombiner() { - return new GenericLastAggregateCombiner>(){}; + return new GenericLastAggregateCombiner(SerializablePairLongLong.class); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetrics.java b/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetrics.java index 4f40ff6bd83d..d79e1a44ce8c 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetrics.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/ComplexMetrics.java @@ -37,6 +37,14 @@ public static ComplexMetricSerde getSerdeForType(String type) return COMPLEX_SERIALIZERS.get(type); } + /** + * Register a serde by its type name + */ + public static void registerSerde(ComplexMetricSerde serde) + { + registerSerde(serde.getTypeName(), serde); + } + /** * Register a serde name -> ComplexMetricSerde mapping. * diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index c937ab7f664d..0d77321fee42 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -156,7 +156,7 @@ public void testDoubleLastRollup() throws Exception public void testFloatFirstRollup() throws Exception { testFirstLastRollup( - doubleEventsList, + floatEventsList, ImmutableList.of("d"), new FloatFirstAggregatorFactory("m", "m") ); @@ -166,7 +166,7 @@ public void testFloatFirstRollup() throws Exception public void testFloatLastRollup() throws Exception { testFirstLastRollup( - doubleEventsList, + floatEventsList, ImmutableList.of("d"), new FloatLastAggregatorFactory("m", "m") ); @@ -176,7 +176,7 @@ public void testFloatLastRollup() throws Exception public void testLongFirstRollup() throws Exception { testFirstLastRollup( - doubleEventsList, + longEventsList, ImmutableList.of("d"), new LongFirstAggregatorFactory("m", "m") ); @@ -186,7 +186,7 @@ public void testLongFirstRollup() throws Exception public void testLongLastRollup() throws Exception { testFirstLastRollup( - doubleEventsList, + longEventsList, ImmutableList.of("d"), new LongLastAggregatorFactory("m", "m") ); From 8bf9e003feefde1b60154587f59aad61e006066a Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 1 Mar 2021 16:07:23 +0800 Subject: [PATCH 04/19] fix IT testcases --- .../AbstractSerializablePairSerde.java | 6 +++--- .../first/DoubleFirstAggregatorFactory.java | 2 ++ .../first/FloatFirstAggregatorFactory.java | 2 ++ .../first/GenericFirstAggregateCombiner.java | 19 +++++-------------- .../first/LongFirstAggregatorFactory.java | 2 ++ .../last/DoubleLastAggregatorFactory.java | 2 ++ .../aggregation/last/FloatLastAggregator.java | 1 - .../last/FloatLastAggregatorFactory.java | 2 ++ .../last/GenericLastAggregateCombiner.java | 17 +++++------------ .../last/LongLastAggregatorFactory.java | 2 ++ 10 files changed, 25 insertions(+), 30 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java index dccb389adede..5ad71bab6696 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java @@ -63,7 +63,7 @@ public Class extractedClass() @Override public T extractValue(InputRow inputRow, String metricName) { - return (T)inputRow.getRaw(metricName); + return (T) inputRow.getRaw(metricName); } }; } @@ -112,7 +112,7 @@ public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWri return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } - abstract protected T toPairObject(ByteBuffer buffer, int numBytes); + protected abstract T toPairObject(ByteBuffer buffer, int numBytes); - abstract protected byte[] pairToBytes(T val); + protected abstract byte[] pairToBytes(T val); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index b10574b68509..4c0e8fd6bc1b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("doubleFirst") public class DoubleFirstAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new DoubleFirstAggregator( diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 11c2fe9ea24e..411a56e42a8e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("floatFirst") public class FloatFirstAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new FloatFirstAggregator( diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java index 9972728274f3..caba5ac003bb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/GenericFirstAggregateCombiner.java @@ -26,16 +26,7 @@ import javax.annotation.Nullable; -/** - * This class is used for FIRST aggregator - *

- * It's marked as abstract to prevent instantiation of this type directly. - * it must be used it as follow: - *

- * AggregateCombiner combiner = new GenericFirstAggregateCombiner>(){ - * }; - */ -final public class GenericFirstAggregateCombiner> +public class GenericFirstAggregateCombiner> extends ObjectAggregateCombiner { private final Class pairClass; @@ -47,13 +38,13 @@ public GenericFirstAggregateCombiner(Class pairClass) } @Override - public final void reset(ColumnValueSelector selector) + public void reset(ColumnValueSelector selector) { firstValue = (T) selector.getObject(); } @Override - public final void fold(ColumnValueSelector selector) + public void fold(ColumnValueSelector selector) { T newValue = (T) selector.getObject(); @@ -64,13 +55,13 @@ public final void fold(ColumnValueSelector selector) @Nullable @Override - public final T getObject() + public T getObject() { return firstValue; } @Override - public final Class classOfObject() + public Class classOfObject() { return this.pairClass; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index 78498f5ce9e1..afba0e5792ce 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; @@ -47,6 +48,7 @@ import java.util.List; import java.util.Map; +@JsonTypeName("longFirst") public class LongFirstAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new LongFirstAggregator( diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 216135618089..2d06e4f7e0fe 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; @@ -50,6 +51,7 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("doubleLast") public class DoubleLastAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new DoubleLastAggregator( diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java index 293c7ca4bb89..2e87c8efa48f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index ad336897c7f9..6257aa52dc92 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; @@ -50,6 +51,7 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("floatLast") public class FloatLastAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new FloatLastAggregator( diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java index 9480dde57242..5fb2f3960fe0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -26,14 +26,7 @@ import javax.annotation.Nullable; -/** - * This class is marked as abstract to prevent instantiation of this type directly, - * because it relyes on type reflection to get the right class object of generic type - *

- * use it as follow: - * AggregateCombiner combiner = new GenericLastAggregateCombiner>(){}; - */ -final public class GenericLastAggregateCombiner> +public class GenericLastAggregateCombiner> extends ObjectAggregateCombiner { private T lastValue; @@ -46,13 +39,13 @@ public GenericLastAggregateCombiner(Class pairClass) } @Override - public final void reset(ColumnValueSelector selector) + public void reset(ColumnValueSelector selector) { lastValue = (T) selector.getObject(); } @Override - public final void fold(ColumnValueSelector selector) + public void fold(ColumnValueSelector selector) { T newValue = (T) selector.getObject(); @@ -63,13 +56,13 @@ public final void fold(ColumnValueSelector selector) @Nullable @Override - public final T getObject() + public T getObject() { return lastValue; } @Override - public final Class classOfObject() + public Class classOfObject() { return this.pairClass; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 6ef12ecbd60a..ffa15642727d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.java.util.common.StringUtils; @@ -49,6 +50,7 @@ import java.util.Map; import java.util.Objects; +@JsonTypeName("longLast") public class LongLastAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new LongLastAggregator( From 32439bcc3faf4f77b98fdf47aff607187b451395 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 3 Mar 2021 15:06:16 +0800 Subject: [PATCH 05/19] fix reindex bug Signed-off-by: frank chen --- .../overlord/SingleTaskBackgroundRunner.java | 8 ++ .../wikipedia_merge_index_queries.json | 40 ++++++- .../indexer/wikipedia_merge_index_task.json | 30 +++++ .../indexer/wikipedia_merge_reindex_task.json | 92 +++++++++++++++ .../AbstractSerializablePairSerde.java | 8 +- .../first/DoubleFirstAggregator.java | 18 ++- .../first/DoubleFirstAggregatorFactory.java | 109 +++++------------ .../first/DoubleFirstBufferAggregator.java | 20 +++- .../first/FloatFirstAggregator.java | 17 ++- .../first/FloatFirstAggregatorFactory.java | 107 +++++------------ .../first/FloatFirstBufferAggregator.java | 21 ++-- .../first/LongFirstAggregator.java | 17 ++- .../first/LongFirstAggregatorFactory.java | 106 +++++------------ .../first/LongFirstBufferAggregator.java | 23 +++- .../first/NumericFirstAggregator.java | 40 +++++-- .../first/NumericFirstBufferAggregator.java | 54 +++++++-- .../first/StringFirstAggregatorFactory.java | 4 +- .../first/StringFirstLastUtils.java | 13 ++- .../last/DoubleLastAggregator.java | 18 ++- .../last/DoubleLastAggregatorFactory.java | 110 +++++------------- .../last/DoubleLastBufferAggregator.java | 17 ++- .../aggregation/last/FloatLastAggregator.java | 18 ++- .../last/FloatLastAggregatorFactory.java | 106 +++++------------ .../last/FloatLastBufferAggregator.java | 17 ++- .../aggregation/last/LongLastAggregator.java | 18 ++- .../last/LongLastAggregatorFactory.java | 106 +++++------------ .../last/LongLastBufferAggregator.java | 21 +++- .../last/NumericLastAggregator.java | 39 +++++-- .../last/NumericLastBufferAggregator.java | 47 ++++++-- .../last/StringLastAggregatorFactory.java | 4 +- 30 files changed, 671 insertions(+), 577 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java index cfbbab4bf3e2..0b3cb84f7120 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java @@ -440,6 +440,14 @@ public TaskStatus call() TaskStatus status; + for (int i = 60; i >= 0; i--) { + log.info("Waiting %s", i); + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + } + } try { log.info("Running task: %s", task.getId()); TaskRunnerUtils.notifyLocationChanged( diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json index ab4674999b5d..b2953d97ebea 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_queries.json @@ -1,6 +1,6 @@ [ { - "description": "groupby, stringFirst/stringLast rollup aggs, all", + "description": "groupby, stringFirst/stringLast/doubleFirst/doubleLast/longFirst/longLast/floatFirst/floatLast rollup aggs, all", "query":{ "queryType" : "groupBy", "dataSource": "%%DATASOURCE%%", @@ -26,6 +26,36 @@ "type":"stringLast", "name":"latest_user", "fieldName":"last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longFirst", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ] }, @@ -35,7 +65,13 @@ "event" : { "continent":"Asia", "earliest_user":"masterYi", - "latest_user":"stringer" + "latest_user":"stringer", + "double_first_delta": 111.0, + "double_last_delta": -9.0, + "long_first_delta": 111, + "long_last_delta": -9, + "float_first_delta": 111.0, + "float_last_delta": -9.0 } } ] } diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json index 43264a8c6751..66379d0d0fa4 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_index_task.json @@ -23,6 +23,36 @@ "name": "delta", "fieldName": "delta" }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "delta" + }, { "type": "stringFirst", "name": "first_user", diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json index 127461dd117c..56690c414d7a 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json @@ -19,6 +19,36 @@ "name": "delta", "fieldName": "delta" }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" + }, { "type": "stringFirst", "name": "first_user", @@ -63,3 +93,65 @@ } } } + +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "inline_data2_reindex4", + "metricsSpec": [ + + { + "type": "longFirst", + "name": "firstDelta", + "fieldName": "firstDelta" + }, + { + "type": "longLast", + "name": "lastDelta", + "fieldName": "lastDelta" + }, + { + "type": "stringLast", + "name": "lastUser", + "fieldName": "lastUser" + }, + { + "type": "stringFirst", + "name": "firstUser", + "fieldName": "firstUser" + } + ], + "granularitySpec": { + "segmentGranularity": "DAY", + "queryGranularity": "DAY", + "intervals": [ + "2013-08-31/2013-09-01" + ] + }, + "timestampSpec": { + "column": "timestamp", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "continent" + ] + } + }, + "ioConfig": { + "type": "index", + "inputSource": { + "type": "ingestSegment", + "dataSource": "inline_data2", + "interval": "2013-08-31/2013-09-01" + }, + "inputFormat": { + "type": "json" + } + }, + "tuningConfig": { + "type": "index" + } + } +} \ No newline at end of file diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java index 5ad71bab6696..86944ab6bcf2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java @@ -50,9 +50,9 @@ public AbstractSerializablePairSerde(Class pairClassObject) } @Override - public ComplexMetricExtractor getExtractor() + public ComplexMetricExtractor getExtractor() { - return new ComplexMetricExtractor() + return new ComplexMetricExtractor() { @Override public Class extractedClass() @@ -61,9 +61,9 @@ public Class extractedClass() } @Override - public T extractValue(InputRow inputRow, String metricName) + public Object extractValue(InputRow inputRow, String metricName) { - return (T) inputRow.getRaw(metricName); + return inputRow.getRaw(metricName); } }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java index e68c7c1ada8c..0942c375e837 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java @@ -20,25 +20,33 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.query.aggregation.SerializablePairLongDouble; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class DoubleFirstAggregator extends NumericFirstAggregator +public class DoubleFirstAggregator extends NumericFirstAggregator { double firstValue; - public DoubleFirstAggregator(BaseLongColumnValueSelector timeSelector, BaseDoubleColumnValueSelector valueSelector) + public DoubleFirstAggregator(BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); firstValue = 0; } @Override - void setCurrentValue() + void setCurrentValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getDouble(); } + @Override + void setCurrentValue(Number number) + { + firstValue = number.doubleValue(); + } + @Override public Object get() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index 4c0e8fd6bc1b..d9c12e37e9c2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -32,8 +32,6 @@ import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -54,7 +52,8 @@ public class DoubleFirstAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new DoubleFirstAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -66,7 +65,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new DoubleFirstBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -100,29 +100,39 @@ public DoubleFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; - } else { - return new DoubleFirstAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new DoubleFirstAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; - } else { - return new DoubleFirstBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new DoubleFirstBufferAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) + ); } @Override @@ -159,68 +169,7 @@ public AggregateCombiner makeAggregateCombiner() @Override public AggregatorFactory getCombiningFactory() { - return new DoubleFirstAggregatorFactory(name, name) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - if (pair.rhs != null) { - firstValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleFirstBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putDouble(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = (SerializablePair) selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new DoubleFirstAggregatorFactory(name, name); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java index dabade475369..2a6ced695719 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java @@ -20,19 +20,21 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class DoubleFirstBufferAggregator extends NumericFirstBufferAggregator +public class DoubleFirstBufferAggregator extends NumericFirstBufferAggregator { public DoubleFirstBufferAggregator( BaseLongColumnValueSelector timeSelector, - BaseDoubleColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -42,16 +44,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putDouble(position, valueSelector.getDouble()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putDouble(position, value.doubleValue()); + } + @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java index a563d7f30955..bbf568f12470 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java @@ -20,28 +20,35 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.query.aggregation.SerializablePairLongFloat; -import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class FloatFirstAggregator extends NumericFirstAggregator +public class FloatFirstAggregator extends NumericFirstAggregator { float firstValue; public FloatFirstAggregator( BaseLongColumnValueSelector timeSelector, - BaseFloatColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); firstValue = 0; } @Override - void setCurrentValue() + void setCurrentValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getFloat(); } + @Override + void setCurrentValue(Number number) + { + firstValue = number.floatValue(); + } + @Override public Object get() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 411a56e42a8e..dc4ebee82c73 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -33,7 +33,6 @@ import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -54,7 +53,8 @@ public class FloatFirstAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new FloatFirstAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -66,7 +66,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new FloatFirstBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -98,29 +99,39 @@ public FloatFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; - } else { - return new FloatFirstAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new FloatFirstAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; - } else { - return new FloatFirstBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new FloatFirstBufferAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) + ); } @Override @@ -157,67 +168,7 @@ public AggregateCombiner makeAggregateCombiner() @Override public AggregatorFactory getCombiningFactory() { - - return new FloatFirstAggregatorFactory(name, name) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - if (pair.rhs != null) { - firstValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatFirstBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putFloat(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new FloatFirstAggregatorFactory(name, name); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java index cf7d272b0085..e4cdfbf11a88 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstBufferAggregator.java @@ -19,20 +19,21 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class FloatFirstBufferAggregator extends NumericFirstBufferAggregator +public class FloatFirstBufferAggregator extends NumericFirstBufferAggregator { public FloatFirstBufferAggregator( BaseLongColumnValueSelector timeSelector, - BaseFloatColumnValueSelector valueSelector + ColumnValueSelector valueSelector, + boolean needsFoldCheck ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -42,16 +43,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putFloat(position, valueSelector.getFloat()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putFloat(position, value.floatValue()); + } + @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java index 2fc74de7c6b8..b76b67bbae0e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java @@ -21,23 +21,32 @@ import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class LongFirstAggregator extends NumericFirstAggregator +public class LongFirstAggregator extends NumericFirstAggregator { long firstValue; - public LongFirstAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongFirstAggregator(BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); firstValue = 0; } @Override - void setCurrentValue() + void setCurrentValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getLong(); } + @Override + void setCurrentValue(Number number) + { + firstValue = number.longValue(); + } + @Override public Object get() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java index afba0e5792ce..30a083e30ec7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -32,8 +32,6 @@ import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -53,7 +51,8 @@ public class LongFirstAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new LongFirstAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -65,7 +64,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new LongFirstBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -97,29 +97,39 @@ public LongFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; - } else { - return new LongFirstAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new LongFirstAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; - } else { - return new LongFirstBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new LongFirstBufferAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) + ); } @Override @@ -156,66 +166,7 @@ public AggregateCombiner makeAggregateCombiner() @Override public AggregatorFactory getCombiningFactory() { - return new LongFirstAggregatorFactory(name, name) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs < firstTime) { - firstTime = pair.lhs; - if (pair.rhs != null) { - firstValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongFirstBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putLong(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long firstTime = buf.getLong(position); - if (pair.lhs < firstTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new LongFirstAggregatorFactory(name, name); } @Override @@ -330,4 +281,5 @@ public String toString() ", fieldName='" + fieldName + '\'' + '}'; } + } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java index 582cda160153..ba41f1da70a6 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstBufferAggregator.java @@ -19,16 +19,21 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class LongFirstBufferAggregator extends NumericFirstBufferAggregator +public class LongFirstBufferAggregator extends NumericFirstBufferAggregator { - public LongFirstBufferAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongFirstBufferAggregator( + BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck + ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -38,16 +43,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putLong(position, valueSelector.getLong()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putLong(position, value.longValue()); + } + @Override public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java index e7b60b9e5c81..08e94254cbf2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java @@ -19,28 +19,34 @@ package org.apache.druid.query.aggregation.first; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; /** * Base type for on heap 'first' aggregator for primitive numeric column selectors */ -public abstract class NumericFirstAggregator implements Aggregator +public abstract class NumericFirstAggregator implements Aggregator { private final boolean useDefault = NullHandling.replaceWithDefault(); private final BaseLongColumnValueSelector timeSelector; - - final TSelector valueSelector; + private final ColumnValueSelector valueSelector; + private final boolean needsFoldCheck; long firstTime; boolean rhsNull; - public NumericFirstAggregator(BaseLongColumnValueSelector timeSelector, TSelector valueSelector) + public NumericFirstAggregator( + BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck + ) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.needsFoldCheck = needsFoldCheck; firstTime = Long.MAX_VALUE; rhsNull = !useDefault; @@ -49,16 +55,36 @@ public NumericFirstAggregator(BaseLongColumnValueSelector timeSelector, TSelecto /** * Store the current primitive typed 'first' value */ - abstract void setCurrentValue(); + abstract void setCurrentValue(ColumnValueSelector valueSelector); + + abstract void setCurrentValue(Number number); @Override public void aggregate() { + if (needsFoldCheck) { + + // Need to read this first (before time), just in case it's a SerializablePairLongString (we don't know; it's + // detected at query time). + final Object object = valueSelector.getObject(); + + if (object instanceof SerializablePair) { + + // cast to Pair to support reindex such as doubleFirst into longFirst + final SerializablePair pair = (SerializablePair) object; + if (pair.lhs < firstTime) { + firstTime = pair.lhs; + setCurrentValue(pair.rhs); + } + return; + } + } + long time = timeSelector.getLong(); if (time < firstTime) { firstTime = time; if (useDefault || !valueSelector.isNull()) { - setCurrentValue(); + setCurrentValue(valueSelector); rhsNull = false; } else { rhsNull = true; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java index ebb0a87169a0..db969f5a47a6 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java @@ -19,18 +19,19 @@ package org.apache.druid.query.aggregation.first; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; /** * Base type for buffer based 'first' aggregator for primitive numeric column selectors */ -public abstract class NumericFirstBufferAggregator +public abstract class NumericFirstBufferAggregator implements BufferAggregator { static final int NULL_OFFSET = Long.BYTES; @@ -38,13 +39,18 @@ public abstract class NumericFirstBufferAggregator to support reindex such as doubleFirst into longFirst + final SerializablePair pair = (SerializablePair) object; + if (pair.lhs < firstTime) { + updateTimeWithValue(buf, position, pair.lhs, pair.rhs); + } + return; + } + } + long time = timeSelector.getLong(); - long firstTime = buf.getLong(position); if (time < firstTime) { if (useDefault || !valueSelector.isNull()) { - updateTimeWithValue(buf, position, time); + updateTimeWithValue(buf, position, time, valueSelector); } else { updateTimeWithNull(buf, position, time); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java index 32a35433da1c..ded2970478ae 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstAggregatorFactory.java @@ -158,7 +158,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } @@ -174,7 +174,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java index 910cb9433581..0ab0bbf4f9aa 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java @@ -36,11 +36,16 @@ public class StringFirstLastUtils private static final int NULL_VALUE = -1; /** - * Returns whether a given value selector *might* contain SerializablePairLongString objects. + * Returns whether a given value selector *might* contain objects with type of + * {@link SerializablePairLongString } + * {@link org.apache.druid.query.aggregation.SerializablePairLongLong} + * {@link org.apache.druid.query.aggregation.SerializablePairLongDouble} + * {@link org.apache.druid.query.aggregation.SerializablePairLongFloat} */ public static boolean selectorNeedsFoldCheck( final BaseObjectColumnValueSelector valueSelector, - @Nullable final ColumnCapabilities valueSelectorCapabilities + @Nullable final ColumnCapabilities valueSelectorCapabilities, + Class objectClass ) { if (valueSelectorCapabilities != null && valueSelectorCapabilities.getType() != ValueType.COMPLEX) { @@ -55,8 +60,8 @@ public static boolean selectorNeedsFoldCheck( // Check if the selector class could possibly be a SerializablePairLongString (either a superclass or subclass). final Class clazz = valueSelector.classOfObject(); - return clazz.isAssignableFrom(SerializablePairLongString.class) - || SerializablePairLongString.class.isAssignableFrom(clazz); + return clazz.isAssignableFrom(objectClass) + || objectClass.isAssignableFrom(clazz); } @Nullable diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java index 213f8085e59e..504e113a8597 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java @@ -20,25 +20,33 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongDouble; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class DoubleLastAggregator extends NumericLastAggregator +public class DoubleLastAggregator extends NumericLastAggregator { double lastValue; - public DoubleLastAggregator(BaseLongColumnValueSelector timeSelector, BaseDoubleColumnValueSelector valueSelector) + public DoubleLastAggregator(BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue() + void setCurrentValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getDouble(); } + @Override + void setCurrentValue(Number number) + { + lastValue = number.doubleValue(); + } + @Override public Object get() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 2d06e4f7e0fe..759966be5c29 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -34,8 +34,7 @@ import org.apache.druid.query.aggregation.SerializablePairLongDoubleSerde; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -56,7 +55,8 @@ public class DoubleLastAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new DoubleLastAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -68,7 +68,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new DoubleLastBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -98,29 +99,39 @@ public DoubleLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; - } else { - return new DoubleLastAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new DoubleLastAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseDoubleColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; - } else { - return new DoubleLastBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new DoubleLastBufferAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongDouble.class + ) + ); } @Override @@ -157,68 +168,7 @@ public AggregateCombiner makeAggregateCombiner() @Override public AggregatorFactory getCombiningFactory() { - return new DoubleLastAggregatorFactory(name, name) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - if (pair.rhs != null) { - lastValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = - metricFactory.makeColumnValueSelector(name); - return new DoubleLastBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putDouble(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new DoubleLastAggregatorFactory(name, name); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java index 8acddce53a83..d8c2f0358b05 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java @@ -20,8 +20,10 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; @@ -29,10 +31,11 @@ public class DoubleLastBufferAggregator extends NumericLastBufferAggregator(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java index 2e87c8efa48f..11a19a56f71f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java @@ -20,25 +20,33 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongFloat; -import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class FloatLastAggregator extends NumericLastAggregator +public class FloatLastAggregator extends NumericLastAggregator { float lastValue; - public FloatLastAggregator(BaseLongColumnValueSelector timeSelector, BaseFloatColumnValueSelector valueSelector) + public FloatLastAggregator(BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue() + void setCurrentValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getFloat(); } + @Override + void setCurrentValue(Number number) + { + lastValue = number.floatValue(); + } + @Override public Object get() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 6257aa52dc92..3fcba9087363 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -34,8 +34,7 @@ import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -56,7 +55,8 @@ public class FloatLastAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new FloatLastAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -68,7 +68,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new FloatLastBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -96,29 +97,39 @@ public FloatLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; - } else { - return new FloatLastAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new FloatLastAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseFloatColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; - } else { - return new FloatLastBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new FloatLastBufferAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongFloat.class + ) + ); } @Override @@ -155,66 +166,7 @@ public AggregateCombiner makeAggregateCombiner() @Override public AggregatorFactory getCombiningFactory() { - return new FloatLastAggregatorFactory(name, name) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - if (pair.rhs != null) { - lastValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new FloatLastBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putFloat(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new FloatLastAggregatorFactory(name, name); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java index 95ad6fe5c5ee..1f12104c634d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java @@ -20,8 +20,10 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; @@ -29,10 +31,11 @@ public class FloatLastBufferAggregator extends NumericLastBufferAggregator(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java index 9d9ebe0fd4ba..598cd473013b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java @@ -21,23 +21,33 @@ import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; -public class LongLastAggregator extends NumericLastAggregator +public class LongLastAggregator extends NumericLastAggregator { long lastValue; - public LongLastAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongLastAggregator(BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck + ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue() + void setCurrentValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getLong(); } + @Override + void setCurrentValue(Number number) + { + lastValue = number.longValue(); + } + @Override public Object get() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index ffa15642727d..b7afae707389 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -33,8 +33,7 @@ import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongLongSerde; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; @@ -55,7 +54,8 @@ public class LongLastAggregatorFactory extends AggregatorFactory { private static final Aggregator NIL_AGGREGATOR = new LongLastAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -67,7 +67,8 @@ public void aggregate() private static final BufferAggregator NIL_BUFFER_AGGREGATOR = new LongLastBufferAggregator( NilColumnValueSelector.instance(), - NilColumnValueSelector.instance() + NilColumnValueSelector.instance(), + false ) { @Override @@ -95,29 +96,39 @@ public LongLastAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; - } else { - return new LongLastAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new LongLastAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + valueSelector, + StringFirstLastUtils.selectorNeedsFoldCheck( + valueSelector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) + ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - final BaseLongColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); - if (valueSelector instanceof NilColumnValueSelector) { + final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); + if (selector instanceof NilColumnValueSelector) { return NIL_BUFFER_AGGREGATOR; - } else { - return new LongLastBufferAggregator( - metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - valueSelector - ); } + + return new LongLastBufferAggregator( + metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), + selector, + StringFirstLastUtils.selectorNeedsFoldCheck( + selector, + metricFactory.getColumnCapabilities(fieldName), + SerializablePairLongLong.class + ) + ); } @Override @@ -154,66 +165,7 @@ public AggregateCombiner makeAggregateCombiner() @Override public AggregatorFactory getCombiningFactory() { - return new LongLastAggregatorFactory(name, name) - { - @Override - public Aggregator factorize(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongLastAggregator(null, null) - { - @Override - public void aggregate() - { - SerializablePair pair = selector.getObject(); - if (pair.lhs >= lastTime) { - lastTime = pair.lhs; - if (pair.rhs != null) { - lastValue = pair.rhs; - rhsNull = false; - } else { - rhsNull = true; - } - } - } - }; - } - - @Override - public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) - { - final ColumnValueSelector> selector = metricFactory.makeColumnValueSelector(name); - return new LongLastBufferAggregator(null, null) - { - @Override - public void putValue(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - buf.putLong(position, pair.rhs); - } - - @Override - public void aggregate(ByteBuffer buf, int position) - { - SerializablePair pair = selector.getObject(); - long lastTime = buf.getLong(position); - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeWithValue(buf, position, pair.lhs); - } else { - updateTimeWithNull(buf, position, pair.lhs); - } - } - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("selector", selector); - } - }; - } - }; + return new LongLastAggregatorFactory(name, name); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java index 981ba3e2f665..d86072ad4899 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java @@ -19,16 +19,21 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; public class LongLastBufferAggregator extends NumericLastBufferAggregator { - public LongLastBufferAggregator(BaseLongColumnValueSelector timeSelector, BaseLongColumnValueSelector valueSelector) + public LongLastBufferAggregator( + BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck + ) { - super(timeSelector, valueSelector); + super(timeSelector, valueSelector, needsFoldCheck); } @Override @@ -38,16 +43,22 @@ void initValue(ByteBuffer buf, int position) } @Override - void putValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, ColumnValueSelector valueSelector) { buf.putLong(position, valueSelector.getLong()); } + @Override + void putValue(ByteBuffer buf, int position, Number value) + { + buf.putLong(position, value.longValue()); + } + @Override public Object get(ByteBuffer buf, int position) { boolean rhsNull = isValueNull(buf, position); - return new SerializablePair<>(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java index 6506f976aeff..39c988e73576 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java @@ -19,29 +19,35 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; /** * Base type for on heap 'last' aggregator for primitive numeric column selectors.. - * + *

* This could probably share a base class with {@link org.apache.druid.query.aggregation.first.NumericFirstAggregator} */ -public abstract class NumericLastAggregator implements Aggregator +public abstract class NumericLastAggregator implements Aggregator { private final boolean useDefault = NullHandling.replaceWithDefault(); private final BaseLongColumnValueSelector timeSelector; - - final TSelector valueSelector; + private final boolean needsFoldCheck; + private final ColumnValueSelector valueSelector; long lastTime; boolean rhsNull; - public NumericLastAggregator(BaseLongColumnValueSelector timeSelector, TSelector valueSelector) + public NumericLastAggregator( + BaseLongColumnValueSelector timeSelector, + ColumnValueSelector valueSelector, + boolean needsFoldCheck + ) { this.timeSelector = timeSelector; this.valueSelector = valueSelector; + this.needsFoldCheck = needsFoldCheck; lastTime = Long.MIN_VALUE; rhsNull = !useDefault; @@ -50,11 +56,26 @@ public NumericLastAggregator(BaseLongColumnValueSelector timeSelector, TSelector @Override public void aggregate() { + if (needsFoldCheck) { + // Need to read this first (before time), just in case it's a SerializablePairLongString (we don't know; it's + // detected at query time). + final Object object = valueSelector.getObject(); + + if (object instanceof SerializablePair) { + final SerializablePair pair = (SerializablePair) object; + if (pair.lhs >= lastTime) { + lastTime = pair.lhs; + setCurrentValue(pair.rhs); + } + return; + } + } + long time = timeSelector.getLong(); if (time >= lastTime) { lastTime = time; if (useDefault || !valueSelector.isNull()) { - setCurrentValue(); + setCurrentValue(valueSelector); rhsNull = false; } else { rhsNull = true; @@ -71,5 +92,7 @@ public void close() /** * Store the current primitive typed 'first' value */ - abstract void setCurrentValue(); + abstract void setCurrentValue(ColumnValueSelector valueSelector); + + abstract void setCurrentValue(Number number); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java index 7c90aadafb5f..04c31699dab2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java @@ -19,11 +19,13 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.BaseNullableColumnValueSelector; +import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; @@ -41,13 +43,16 @@ public abstract class NumericLastBufferAggregator to support reindex such as doubleLast into longLast + final SerializablePair pair = (SerializablePair) object; + if (pair.lhs >= lastTime) { + updateTimeWithValue(buf, position, pair.lhs, pair.rhs); + } + return; + } + } + long time = timeSelector.getLong(); - long lastTime = buf.getLong(position); + if (time >= lastTime) { if (useDefault || !valueSelector.isNull()) { - updateTimeWithValue(buf, position, time); + updateTimeWithValue(buf, position, time, valueSelector); } else { updateTimeWithNull(buf, position, time); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java index 29543518f66c..6c8576e2b468 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastAggregatorFactory.java @@ -115,7 +115,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } @@ -131,7 +131,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), valueSelector, maxStringBytes, - StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName)) + StringFirstLastUtils.selectorNeedsFoldCheck(valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongString.class) ); } } From 48e196f7e59f9ded9e116d322f0410ee50f63053 Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 4 Mar 2021 17:25:55 +0800 Subject: [PATCH 06/19] fix doc and UT/IT Signed-off-by: frank chen --- ...merge_reindex_druid_input_source_task.json | 30 +++++++++ .../indexer/wikipedia_merge_reindex_task.json | 62 ------------------- .../AbstractSerializablePairSerde.java | 4 +- .../first/DoubleFirstAggregatorFactory.java | 8 +-- .../first/DoubleFirstBufferAggregator.java | 6 +- .../first/FloatFirstAggregatorFactory.java | 1 - .../first/StringFirstLastUtils.java | 16 ++--- .../last/DoubleLastBufferAggregator.java | 6 +- .../last/FloatLastBufferAggregator.java | 1 - .../aggregation/AggregatorFactoryTest.java | 12 ++-- .../first/DoubleFirstAggregationTest.java | 7 ++- .../first/FloatFirstAggregationTest.java | 4 ++ .../first/LongFirstAggregationTest.java | 4 ++ .../last/DoubleLastAggregationTest.java | 4 ++ .../last/FloatLastAggregationTest.java | 4 ++ .../last/LongLastAggregationTest.java | 4 ++ 16 files changed, 82 insertions(+), 91 deletions(-) diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json index 9daae62c8d42..348aff886455 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_druid_input_source_task.json @@ -56,6 +56,36 @@ "type": "stringLast", "name": "last_user", "fieldName": "last_user" + }, + { + "type": "doubleFirst", + "name": "double_first_delta", + "fieldName": "double_first_delta" + }, + { + "type": "doubleLast", + "name": "double_last_delta", + "fieldName": "double_last_delta" + }, + { + "type": "longFirst", + "name": "long_first_delta", + "fieldName": "long_first_delta" + }, + { + "type": "longLast", + "name": "long_last_delta", + "fieldName": "long_last_delta" + }, + { + "type": "floatFirst", + "name": "float_first_delta", + "fieldName": "float_first_delta" + }, + { + "type": "floatLast", + "name": "float_last_delta", + "fieldName": "float_last_delta" } ] } diff --git a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json index 56690c414d7a..b0f9959a476b 100644 --- a/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json +++ b/integration-tests/src/test/resources/indexer/wikipedia_merge_reindex_task.json @@ -92,66 +92,4 @@ "type": "index" } } -} - -{ - "type": "index", - "spec": { - "dataSchema": { - "dataSource": "inline_data2_reindex4", - "metricsSpec": [ - - { - "type": "longFirst", - "name": "firstDelta", - "fieldName": "firstDelta" - }, - { - "type": "longLast", - "name": "lastDelta", - "fieldName": "lastDelta" - }, - { - "type": "stringLast", - "name": "lastUser", - "fieldName": "lastUser" - }, - { - "type": "stringFirst", - "name": "firstUser", - "fieldName": "firstUser" - } - ], - "granularitySpec": { - "segmentGranularity": "DAY", - "queryGranularity": "DAY", - "intervals": [ - "2013-08-31/2013-09-01" - ] - }, - "timestampSpec": { - "column": "timestamp", - "format": "iso" - }, - "dimensionsSpec": { - "dimensions": [ - "continent" - ] - } - }, - "ioConfig": { - "type": "index", - "inputSource": { - "type": "ingestSegment", - "dataSource": "inline_data2", - "interval": "2013-08-31/2013-09-01" - }, - "inputFormat": { - "type": "json" - } - }, - "tuningConfig": { - "type": "index" - } - } } \ No newline at end of file diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java index 86944ab6bcf2..e8f2d2641c21 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java @@ -36,9 +36,7 @@ import java.nio.ByteBuffer; /** - * The class serializes a Long-X pair object - *

- * Note: T must be a concrete class which inherits from SerializablePair, such as {@link SerializablePairLongDouble} + * The class serializes a Pair object for double/float/longFirst and double/float/longLast aggregators */ public abstract class AbstractSerializablePairSerde> extends ComplexMetricSerde { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index d9c12e37e9c2..f7f00599d168 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -100,16 +100,16 @@ public DoubleFirstAggregatorFactory( @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - final ColumnValueSelector selector = metricFactory.makeColumnValueSelector(fieldName); - if (selector instanceof NilColumnValueSelector) { + final ColumnValueSelector valueSelector = metricFactory.makeColumnValueSelector(fieldName); + if (valueSelector instanceof NilColumnValueSelector) { return NIL_AGGREGATOR; } return new DoubleFirstAggregator( metricFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME), - selector, + valueSelector, StringFirstLastUtils.selectorNeedsFoldCheck( - selector, + valueSelector, metricFactory.getColumnCapabilities(fieldName), SerializablePairLongDouble.class ) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java index 2a6ced695719..9b56104b7192 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstBufferAggregator.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation.first; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnValueSelector; @@ -59,7 +58,10 @@ void putValue(ByteBuffer buf, int position, Number value) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble( + buf.getLong(position), + rhsNull ? null : buf.getDouble(position + VALUE_OFFSET) + ); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index dc4ebee82c73..d7fe64225d5f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -32,7 +32,6 @@ import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongFloatSerde; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.NilColumnValueSelector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java index 0ab0bbf4f9aa..ad8ba7fe993f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java @@ -36,16 +36,16 @@ public class StringFirstLastUtils private static final int NULL_VALUE = -1; /** - * Returns whether a given value selector *might* contain objects with type of - * {@link SerializablePairLongString } - * {@link org.apache.druid.query.aggregation.SerializablePairLongLong} - * {@link org.apache.druid.query.aggregation.SerializablePairLongDouble} - * {@link org.apache.druid.query.aggregation.SerializablePairLongFloat} + * Returns whether a given value selector *might* contain objects with given type + * + * @param pairClass should be one of the following {@link SerializablePairLongString } {@link org.apache.druid.query.aggregation.SerializablePairLongLong} + * {@link org.apache.druid.query.aggregation.SerializablePairLongDouble} + * {@link org.apache.druid.query.aggregation.SerializablePairLongFloat} */ public static boolean selectorNeedsFoldCheck( final BaseObjectColumnValueSelector valueSelector, @Nullable final ColumnCapabilities valueSelectorCapabilities, - Class objectClass + Class pairClass ) { if (valueSelectorCapabilities != null && valueSelectorCapabilities.getType() != ValueType.COMPLEX) { @@ -60,8 +60,8 @@ public static boolean selectorNeedsFoldCheck( // Check if the selector class could possibly be a SerializablePairLongString (either a superclass or subclass). final Class clazz = valueSelector.classOfObject(); - return clazz.isAssignableFrom(objectClass) - || objectClass.isAssignableFrom(clazz); + return clazz.isAssignableFrom(pairClass) + || pairClass.isAssignableFrom(clazz); } @Nullable diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java index d8c2f0358b05..9d586219b32a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; @@ -60,7 +59,10 @@ void putValue(ByteBuffer buf, int position, Number value) public Object get(ByteBuffer buf, int position) { final boolean rhsNull = isValueNull(buf, position); - return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble( + buf.getLong(position), + rhsNull ? null : buf.getDouble(position + VALUE_OFFSET) + ); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java index 1f12104c634d..b039b34a4810 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation.last; -import org.apache.druid.collections.SerializablePair; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java index 3aa9aa3ffeb8..3136aa04cb22 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java @@ -209,21 +209,21 @@ public void testResultArraySignature() .add("longSum", ValueType.LONG) .add("longMin", ValueType.LONG) .add("longMax", ValueType.LONG) - .add("longFirst", ValueType.LONG) - .add("longLast", ValueType.LONG) + .add("longFirst", null) + .add("longLast", null) .add("longAny", ValueType.LONG) .add("doubleSum", ValueType.DOUBLE) .add("doubleMin", ValueType.DOUBLE) .add("doubleMax", ValueType.DOUBLE) - .add("doubleFirst", ValueType.DOUBLE) - .add("doubleLast", ValueType.DOUBLE) + .add("doubleFirst", null) + .add("doubleLast", null) .add("doubleAny", ValueType.DOUBLE) .add("doubleMean", null) .add("floatSum", ValueType.FLOAT) .add("floatMin", ValueType.FLOAT) .add("floatMax", ValueType.FLOAT) - .add("floatFirst", ValueType.FLOAT) - .add("floatLast", ValueType.FLOAT) + .add("floatFirst", null) + .add("floatLast", null) .add("floatAny", ValueType.FLOAT) .add("stringFirst", null) .add("stringLast", null) diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 8b101c1ce9ba..153156048993 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -30,7 +30,9 @@ import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -68,8 +70,9 @@ public void setup() objectSelector = new TestObjectColumnSelector<>(pairs); colSelectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); - EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector).atLeastOnce(); + EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector).atLeastOnce(); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.DOUBLE)); EasyMock.replay(colSelectorFactory); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java index f3675cbb4b6c..06e0b6bc89ca 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -30,7 +30,9 @@ import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,6 +72,8 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector).atLeastOnce(); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector).atLeastOnce(); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.FLOAT)); EasyMock.replay(colSelectorFactory); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java index d2f5f914b4de..e2d83af1b2b6 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -29,7 +29,9 @@ import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -69,6 +71,8 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.LONG)); EasyMock.replay(colSelectorFactory); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java index 3d514ea1131f..e349a5370915 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -30,7 +30,9 @@ import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,6 +72,8 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.DOUBLE)); EasyMock.replay(colSelectorFactory); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java index 976d1143c39a..69884fd745e6 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -30,7 +30,9 @@ import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -70,6 +72,8 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.FLOAT)); EasyMock.replay(colSelectorFactory); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java index d4d604108e91..249f7716fa9f 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java @@ -29,7 +29,9 @@ import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.testing.InitializedNullHandlingTest; import org.easymock.EasyMock; import org.junit.Assert; @@ -69,6 +71,8 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.LONG)); EasyMock.replay(colSelectorFactory); } From cfc6814a6dedbd32bd6db1ce4a82f7badc7de557 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 5 Mar 2021 16:42:53 +0800 Subject: [PATCH 07/19] Revert "Web console: Remove first / last suggestions (#10794)" This reverts commit 2a1e47afc3aaea5addf16b956c75a8dbced4cce7. --- web-console/src/druid-models/metric-spec.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/web-console/src/druid-models/metric-spec.tsx b/web-console/src/druid-models/metric-spec.tsx index e72d114cff35..fd1282fe7702 100644 --- a/web-console/src/druid-models/metric-spec.tsx +++ b/web-console/src/druid-models/metric-spec.tsx @@ -67,10 +67,14 @@ export const METRIC_SPEC_FIELDS: Field[] = [ group: 'max', suggestions: ['longMax', 'doubleMax', 'floatMax'], }, - // Do not show first and last aggregators as they can not be used in ingestion specs and this definition is only used in the data loader. - // Ref: https://druid.apache.org/docs/latest/querying/aggregations.html#first--last-aggregator - // Should the first / last aggregators become usable at ingestion time, reverse the changes made in: - // https://github.com/apache/druid/pull/10794 + { + group: 'first', + suggestions: ['longFirst', 'doubleFirst', 'floatFirst'], + }, + { + group: 'last', + suggestions: ['longLast', 'doubleLast', 'floatLast'], + }, 'thetaSketch', { group: 'HLLSketch', From 14ccec1b041c512ded84b4486e4c3328cbc5225c Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 5 Mar 2021 17:04:10 +0800 Subject: [PATCH 08/19] revert unnecessary commit Signed-off-by: frank chen --- .../indexing/overlord/SingleTaskBackgroundRunner.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java index 0b3cb84f7120..cfbbab4bf3e2 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunner.java @@ -440,14 +440,6 @@ public TaskStatus call() TaskStatus status; - for (int i = 60; i >= 0; i--) { - log.info("Waiting %s", i); - try { - Thread.sleep(1000); - } - catch (InterruptedException e) { - } - } try { log.info("Running task: %s", task.getId()); TaskRunnerUtils.notifyLocationChanged( From 998567f2deab7ad32b181cd2c1d51cd05af34de2 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 5 Mar 2021 17:08:13 +0800 Subject: [PATCH 09/19] update doc Signed-off-by: frank chen --- .../query/aggregation/SerializablePairLongDoubleSerde.java | 2 +- .../druid/query/aggregation/SerializablePairLongFloatSerde.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 1910bad74d86..4e53b02a4d7e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -26,7 +26,7 @@ * The serialization structure is: Long:Double *

* The class is used on first/last Double aggregators to store the time and the first/last Double. - * Long:Long -> Timestamp:Long + * Long:Double -> Timestamp:Double */ public class SerializablePairLongDoubleSerde extends AbstractSerializablePairSerde { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index 87f277aa4c1d..995658e34345 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -26,7 +26,7 @@ * The serialization structure is: Long:Float *

* The class is used on first/last Float aggregators to store the time and the first/last Float. - * Long:Long -> Timestamp:Long + * Long:Float -> Timestamp:Float */ public class SerializablePairLongFloatSerde extends AbstractSerializablePairSerde { From a1ffc8eaa7825101839e6174cf55c3eb96514821 Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 8 Mar 2021 17:06:53 +0800 Subject: [PATCH 10/19] fix CI Signed-off-by: frank chen --- .../SerializablePairLongDoubleSerde.java | 19 ++++++++++++++---- .../SerializablePairLongFloatSerde.java | 20 ++++++++++++++----- .../SerializablePairLongLongSerde.java | 20 ++++++++++++++----- .../first/DoubleFirstAggregationTest.java | 14 +++++++++++-- .../first/FloatFirstAggregationTest.java | 19 +++++++++++++++--- .../first/LongFirstAggregationTest.java | 19 ++++++++++++++---- .../last/DoubleLastAggregationTest.java | 19 +++++++++++++++--- .../last/FloatLastAggregationTest.java | 17 +++++++++++++--- .../last/LongLastAggregationTest.java | 19 +++++++++++++++--- 9 files changed, 134 insertions(+), 32 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 4e53b02a4d7e..1c102b50fa47 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import java.nio.ByteBuffer; /** @@ -48,16 +50,25 @@ protected SerializablePairLongDouble toPairObject(ByteBuffer buffer, int numByte { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); - double rhs = readOnlyBuffer.getDouble(); - return new SerializablePairLongDouble(lhs, rhs); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); + } else { + return new SerializablePairLongDouble(lhs, null); + } } @Override protected byte[] pairToBytes(SerializablePairLongDouble val) { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Double.BYTES); + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); bbuf.putLong(val.lhs); - bbuf.putDouble(val.rhs); + if (val.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putDouble(val.rhs); + } return bbuf.array(); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index 995658e34345..416b5297f6e7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import java.nio.ByteBuffer; /** @@ -48,17 +50,25 @@ protected SerializablePairLongFloat toPairObject(ByteBuffer buffer, int numBytes { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); - float rhs = readOnlyBuffer.getFloat(); - return new SerializablePairLongFloat(lhs, rhs); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); + } else { + return new SerializablePairLongFloat(lhs, null); + } } @Override protected byte[] pairToBytes(SerializablePairLongFloat val) { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Float.BYTES); + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); bbuf.putLong(val.lhs); - bbuf.putFloat(val.rhs); + if (val.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putFloat(val.rhs); + } return bbuf.array(); } - } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index cbbd7303a817..fde1866a014d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.common.config.NullHandling; + import java.nio.ByteBuffer; /** @@ -48,17 +50,25 @@ protected SerializablePairLongLong toPairObject(ByteBuffer buffer, int numBytes) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); - long rhs = readOnlyBuffer.getLong(); - return new SerializablePairLongLong(lhs, rhs); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); + } else { + return new SerializablePairLongLong(lhs, null); + } } @Override protected byte[] pairToBytes(SerializablePairLongLong val) { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Long.BYTES); + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); bbuf.putLong(val.lhs); - bbuf.putLong(val.rhs); + if (val.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putLong(val.rhs); + } return bbuf.array(); } - } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java index 153156048993..14f8d871a0b5 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java @@ -72,13 +72,14 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector).atLeastOnce(); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector).atLeastOnce(); - EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.DOUBLE)); - EasyMock.replay(colSelectorFactory); } @Test public void testDoubleFirstAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.DOUBLE)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = doubleFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -97,6 +98,9 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.DOUBLE)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = doubleFirstAggFactory.factorizeBuffered( colSelectorFactory); @@ -152,6 +156,9 @@ public void testComparatorWithNulls() @Test public void testDoubleFirstCombiningAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -171,6 +178,9 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java index 06e0b6bc89ca..f68b1a8e016e 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -72,14 +72,15 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector).atLeastOnce(); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector).atLeastOnce(); - EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( - ValueType.FLOAT)); - EasyMock.replay(colSelectorFactory); } @Test public void testDoubleFirstAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.FLOAT)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = floatFirstAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); @@ -98,6 +99,10 @@ public void testDoubleFirstAggregator() @Test public void testDoubleFirstBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.FLOAT)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = floatFirstAggregatorFactory.factorizeBuffered( colSelectorFactory); @@ -140,6 +145,10 @@ public void testComparatorWithNulls() @Test public void testDoubleFirstCombiningAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -159,6 +168,10 @@ public void testDoubleFirstCombiningAggregator() @Test public void testDoubleFirstCombiningBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java index e2d83af1b2b6..07e5dff0a47e 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java @@ -71,14 +71,15 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); - EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( - ValueType.LONG)); - EasyMock.replay(colSelectorFactory); } @Test public void testLongFirstAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.LONG)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = longFirstAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -97,6 +98,10 @@ public void testLongFirstAggregator() @Test public void testLongFirstBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.LONG)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = longFirstAggFactory.factorizeBuffered( colSelectorFactory); @@ -139,6 +144,10 @@ public void testComparatorWithNulls() @Test public void testLongFirstCombiningAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -158,6 +167,9 @@ public void testLongFirstCombiningAggregator() @Test public void testLongFirstCombiningBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType(ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); @@ -178,7 +190,6 @@ public void testLongFirstCombiningBufferAggregator() Assert.assertEquals(expected.rhs, agg.getFloat(buffer, 0), 0.0001); } - @Test public void testSerde() throws Exception { diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java index e349a5370915..62fb84823bc9 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java @@ -72,14 +72,15 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); - EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( - ValueType.DOUBLE)); - EasyMock.replay(colSelectorFactory); } @Test public void testDoubleLastAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.DOUBLE)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = doubleLastAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -98,6 +99,10 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.DOUBLE)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = doubleLastAggFactory.factorizeBuffered( colSelectorFactory); @@ -140,6 +145,10 @@ public void testComparatorWithNulls() @Test public void testDoubleLastCombiningAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -159,6 +168,10 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java index 69884fd745e6..b5d4e11e6967 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -72,14 +72,14 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); - EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( - ValueType.FLOAT)); - EasyMock.replay(colSelectorFactory); } @Test public void testDoubleLastAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.FLOAT)); + EasyMock.replay(colSelectorFactory); Aggregator agg = floatLastAggregatorFactory.factorize(colSelectorFactory); aggregate(agg); @@ -98,6 +98,9 @@ public void testDoubleLastAggregator() @Test public void testDoubleLastBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.FLOAT)); + EasyMock.replay(colSelectorFactory); BufferAggregator agg = floatLastAggregatorFactory.factorizeBuffered( colSelectorFactory); @@ -140,6 +143,10 @@ public void testComparatorWithNulls() @Test public void testDoubleLastCombiningAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -159,6 +166,10 @@ public void testDoubleLastCombiningAggregator() @Test public void testDoubleLastCombiningBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java index 249f7716fa9f..80ddbaef0d26 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java @@ -71,14 +71,15 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME)).andReturn(timeSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("nilly")).andReturn(valueSelector); EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector); - EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( - ValueType.LONG)); - EasyMock.replay(colSelectorFactory); } @Test public void testLongLastAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.LONG)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = longLastAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -97,6 +98,10 @@ public void testLongLastAggregator() @Test public void testLongLastBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("nilly")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.LONG)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = longLastAggFactory.factorizeBuffered( colSelectorFactory); @@ -139,6 +144,10 @@ public void testComparatorWithNulls() @Test public void testLongLastCombiningAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); aggregate(agg); @@ -158,6 +167,10 @@ public void testLongLastCombiningAggregator() @Test public void testLongLastCombiningBufferAggregator() { + EasyMock.expect(colSelectorFactory.getColumnCapabilities("billy")).andReturn(new ColumnCapabilitiesImpl().setType( + ValueType.COMPLEX)); + EasyMock.replay(colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); From cffff8074d412836239b86781614d518434cffed Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 12 Mar 2021 13:24:31 +0800 Subject: [PATCH 11/19] resolve problems in SQL-Compatibility mode --- .../aggregation/AbstractSerializablePairSerde.java | 4 ++-- .../aggregation/SerializablePairLongDoubleSerde.java | 2 +- .../aggregation/SerializablePairLongFloatSerde.java | 2 +- .../aggregation/SerializablePairLongLongSerde.java | 2 +- .../aggregation/first/NumericFirstAggregator.java | 12 ++++++++++-- .../first/NumericFirstBufferAggregator.java | 7 ++++++- .../aggregation/last/DoubleLastBufferAggregator.java | 3 +-- .../aggregation/last/FloatLastBufferAggregator.java | 3 +-- .../aggregation/last/LongLastBufferAggregator.java | 2 +- .../aggregation/last/NumericLastAggregator.java | 11 ++++++++++- .../last/NumericLastBufferAggregator.java | 10 +++++++--- 11 files changed, 41 insertions(+), 17 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java index e8f2d2641c21..6793265fb5f2 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java @@ -93,7 +93,7 @@ public Class getClazz() @Override public T fromByteBuffer(ByteBuffer buffer, int numBytes) { - return toPairObject(buffer, numBytes); + return toPairObject(buffer); } @Override @@ -110,7 +110,7 @@ public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWri return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } - protected abstract T toPairObject(ByteBuffer buffer, int numBytes); + protected abstract T toPairObject(ByteBuffer buffer); protected abstract byte[] pairToBytes(T val); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 1c102b50fa47..2e2b2f07939c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -46,7 +46,7 @@ public String getTypeName() } @Override - protected SerializablePairLongDouble toPairObject(ByteBuffer buffer, int numBytes) + protected SerializablePairLongDouble toPairObject(ByteBuffer buffer) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index 416b5297f6e7..3f26c2f80839 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -46,7 +46,7 @@ public String getTypeName() } @Override - protected SerializablePairLongFloat toPairObject(ByteBuffer buffer, int numBytes) + protected SerializablePairLongFloat toPairObject(ByteBuffer buffer) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index fde1866a014d..a8ba982da2e9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -46,7 +46,7 @@ public String getTypeName() } @Override - protected SerializablePairLongLong toPairObject(ByteBuffer buffer, int numBytes) + protected SerializablePairLongLong toPairObject(ByteBuffer buffer) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java index 08e94254cbf2..444bcc39bb46 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java @@ -70,12 +70,20 @@ public void aggregate() if (object instanceof SerializablePair) { - // cast to Pair to support reindex such as doubleFirst into longFirst + // cast to Pair to support reindex from type such as doubleFirst into longFirst final SerializablePair pair = (SerializablePair) object; if (pair.lhs < firstTime) { firstTime = pair.lhs; - setCurrentValue(pair.rhs); + + // rhs might be NULL under SQL-compatibility mode + if (pair.rhs == null) { + rhsNull = true; + } else { + rhsNull = false; + setCurrentValue(pair.rhs); + } } + return; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java index db969f5a47a6..ae878e5de90b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstBufferAggregator.java @@ -113,7 +113,12 @@ public void aggregate(ByteBuffer buf, int position) // cast to Pair to support reindex such as doubleFirst into longFirst final SerializablePair pair = (SerializablePair) object; if (pair.lhs < firstTime) { - updateTimeWithValue(buf, position, pair.lhs, pair.rhs); + if (pair.rhs == null) { + // rhs might be NULL under SQL-compatibility mode + updateTimeWithNull(buf, position, pair.lhs); + } else { + updateTimeWithValue(buf, position, pair.lhs, pair.rhs); + } } return; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java index 9d586219b32a..74c46d69aaad 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastBufferAggregator.java @@ -20,13 +20,12 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongDouble; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class DoubleLastBufferAggregator extends NumericLastBufferAggregator +public class DoubleLastBufferAggregator extends NumericLastBufferAggregator { public DoubleLastBufferAggregator( BaseLongColumnValueSelector timeSelector, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java index b039b34a4810..4cf443707af7 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastBufferAggregator.java @@ -20,13 +20,12 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongFloat; -import org.apache.druid.segment.BaseFloatColumnValueSelector; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; -public class FloatLastBufferAggregator extends NumericLastBufferAggregator +public class FloatLastBufferAggregator extends NumericLastBufferAggregator { public FloatLastBufferAggregator( BaseLongColumnValueSelector timeSelector, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java index d86072ad4899..80553f4fcc0a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastBufferAggregator.java @@ -25,7 +25,7 @@ import java.nio.ByteBuffer; -public class LongLastBufferAggregator extends NumericLastBufferAggregator +public class LongLastBufferAggregator extends NumericLastBufferAggregator { public LongLastBufferAggregator( BaseLongColumnValueSelector timeSelector, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java index 39c988e73576..220da0691372 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java @@ -62,10 +62,19 @@ public void aggregate() final Object object = valueSelector.getObject(); if (object instanceof SerializablePair) { + + // cast to Pair to support reindex from type such as doubleFirst into another type(longFirst) final SerializablePair pair = (SerializablePair) object; if (pair.lhs >= lastTime) { lastTime = pair.lhs; - setCurrentValue(pair.rhs); + + // rhs might be NULL under SQL-compatibility mode + if (pair.rhs == null) { + rhsNull = true; + } else { + rhsNull = false; + setCurrentValue(pair.rhs); + } } return; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java index 04c31699dab2..3cfd1eb040f9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java @@ -24,7 +24,6 @@ import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseNullableColumnValueSelector; import org.apache.druid.segment.ColumnValueSelector; import java.nio.ByteBuffer; @@ -35,7 +34,7 @@ * This could probably share a base type with * {@link org.apache.druid.query.aggregation.first.NumericFirstBufferAggregator} ... */ -public abstract class NumericLastBufferAggregator +public abstract class NumericLastBufferAggregator implements BufferAggregator { static final int NULL_OFFSET = Long.BYTES; @@ -114,7 +113,12 @@ public void aggregate(ByteBuffer buf, int position) // cast to Pair to support reindex such as doubleLast into longLast final SerializablePair pair = (SerializablePair) object; if (pair.lhs >= lastTime) { - updateTimeWithValue(buf, position, pair.lhs, pair.rhs); + if (pair.rhs == null ) { + // rhs might be NULL under SQL-compatibility mode + updateTimeWithNull(buf, position, pair.lhs); + } else { + updateTimeWithValue(buf, position, pair.lhs, pair.rhs); + } } return; } From f96aa6c9a936056a953bb0928b5fd5e719b7f44e Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 12 Mar 2021 13:30:43 +0800 Subject: [PATCH 12/19] resolve problems in SQL-Compatibility mode --- .../query/aggregation/last/NumericLastBufferAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java index 3cfd1eb040f9..f416255da230 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastBufferAggregator.java @@ -113,7 +113,7 @@ public void aggregate(ByteBuffer buf, int position) // cast to Pair to support reindex such as doubleLast into longLast final SerializablePair pair = (SerializablePair) object; if (pair.lhs >= lastTime) { - if (pair.rhs == null ) { + if (pair.rhs == null) { // rhs might be NULL under SQL-compatibility mode updateTimeWithNull(buf, position, pair.lhs); } else { From 72a7a0d4f906ec0ab61a798dda12f81ecaec3abd Mon Sep 17 00:00:00 2001 From: frank chen Date: Sat, 10 Apr 2021 23:02:46 +0800 Subject: [PATCH 13/19] Address comments --- ...tractSerializableLongObjectPairSerde.java} | 38 ++++++++++++++----- .../SerializablePairLongDoubleSerde.java | 29 +++++++++++--- .../SerializablePairLongFloatSerde.java | 29 +++++++++++--- .../SerializablePairLongLongSerde.java | 29 +++++++++++--- 4 files changed, 97 insertions(+), 28 deletions(-) rename processing/src/main/java/org/apache/druid/query/aggregation/{AbstractSerializablePairSerde.java => AbstractSerializableLongObjectPairSerde.java} (72%) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java similarity index 72% rename from processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java rename to processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java index 6793265fb5f2..e9b08ac4867c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializablePairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java @@ -19,7 +19,6 @@ package org.apache.druid.query.aggregation; -import com.google.common.primitives.Longs; import org.apache.druid.collections.SerializablePair; import org.apache.druid.data.input.InputRow; import org.apache.druid.segment.GenericColumnSerializer; @@ -34,15 +33,17 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.Comparator; /** - * The class serializes a Pair object for double/float/longFirst and double/float/longLast aggregators + * The class serializes/deserializes a Pair object for double/float/longFirst and double/float/longLast aggregators */ -public abstract class AbstractSerializablePairSerde> extends ComplexMetricSerde +public abstract class AbstractSerializableLongObjectPairSerde> + extends ComplexMetricSerde { private final Class pairClassObject; - public AbstractSerializablePairSerde(Class pairClassObject) + public AbstractSerializableLongObjectPairSerde(Class pairClassObject) { this.pairClassObject = pairClassObject; } @@ -81,7 +82,7 @@ public ObjectStrategy getObjectStrategy() @Override public int compare(@Nullable T o1, @Nullable T o2) { - return Longs.compare(o1.lhs, o2.lhs); + return getLongObjectPairComparator().compare(o1, o2); } @Override @@ -93,13 +94,17 @@ public Class getClazz() @Override public T fromByteBuffer(ByteBuffer buffer, int numBytes) { - return toPairObject(buffer); + return toLongObjectPair(buffer); } @Override - public byte[] toBytes(T val) + public byte[] toBytes(@Nullable T val) { - return pairToBytes(val); + if (val == null) { + return new byte[]{}; + } + + return longObjectPairToBytes(val); } }; } @@ -110,7 +115,20 @@ public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWri return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } - protected abstract T toPairObject(ByteBuffer buffer); + /** + * get comparator for Pair object + * It's recommended to use {@link SerializablePair#createNullHandlingComparator(Comparator, boolean)} to instance a concret comparator. + * + */ + protected abstract Comparator getLongObjectPairComparator(); + + /** + * deserialize a Pair object from buffer + */ + protected abstract T toLongObjectPair(ByteBuffer buffer); - protected abstract byte[] pairToBytes(T val); + /** + * serialize a Pair object to byte array + */ + protected abstract byte[] longObjectPairToBytes(T longObjectPair); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 2e2b2f07939c..2f8bfafc5ace 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -19,9 +19,11 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import java.nio.ByteBuffer; +import java.util.Comparator; /** * The class serializes a Long-Double pair (SerializablePair). @@ -30,10 +32,19 @@ * The class is used on first/last Double aggregators to store the time and the first/last Double. * Long:Double -> Timestamp:Double */ -public class SerializablePairLongDoubleSerde extends AbstractSerializablePairSerde +public class SerializablePairLongDoubleSerde extends AbstractSerializableLongObjectPairSerde { public static final String TYPE_NAME = "serializablePairLongDouble"; + /** + * Since SerializablePairLongDouble is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. + */ + public static Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + Double::compare, + true + ); + public SerializablePairLongDoubleSerde() { super(SerializablePairLongDouble.class); @@ -46,7 +57,13 @@ public String getTypeName() } @Override - protected SerializablePairLongDouble toPairObject(ByteBuffer buffer) + protected Comparator getLongObjectPairComparator() + { + return VALUE_COMPARATOR; + } + + @Override + protected SerializablePairLongDouble toLongObjectPair(ByteBuffer buffer) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); @@ -59,15 +76,15 @@ protected SerializablePairLongDouble toPairObject(ByteBuffer buffer) } @Override - protected byte[] pairToBytes(SerializablePairLongDouble val) + protected byte[] longObjectPairToBytes(SerializablePairLongDouble longObjectPair) { ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); - bbuf.putLong(val.lhs); - if (val.rhs == null) { + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { bbuf.put(NullHandling.IS_NULL_BYTE); } else { bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putDouble(val.rhs); + bbuf.putDouble(longObjectPair.rhs); } return bbuf.array(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index 3f26c2f80839..cc00e8c8ac36 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -19,9 +19,11 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import java.nio.ByteBuffer; +import java.util.Comparator; /** * The class serializes a Long-Float pair (SerializablePair). @@ -30,10 +32,19 @@ * The class is used on first/last Float aggregators to store the time and the first/last Float. * Long:Float -> Timestamp:Float */ -public class SerializablePairLongFloatSerde extends AbstractSerializablePairSerde +public class SerializablePairLongFloatSerde extends AbstractSerializableLongObjectPairSerde { public static final String TYPE_NAME = "serializablePairLongFloat"; + /** + * Since SerializablePairLongFloat is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. + */ + public static Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + Float::compare, + true + ); + public SerializablePairLongFloatSerde() { super(SerializablePairLongFloat.class); @@ -46,7 +57,13 @@ public String getTypeName() } @Override - protected SerializablePairLongFloat toPairObject(ByteBuffer buffer) + protected Comparator getLongObjectPairComparator() + { + return VALUE_COMPARATOR; + } + + @Override + protected SerializablePairLongFloat toLongObjectPair(ByteBuffer buffer) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); @@ -59,15 +76,15 @@ protected SerializablePairLongFloat toPairObject(ByteBuffer buffer) } @Override - protected byte[] pairToBytes(SerializablePairLongFloat val) + protected byte[] longObjectPairToBytes(SerializablePairLongFloat longObjectPair) { ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); - bbuf.putLong(val.lhs); - if (val.rhs == null) { + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { bbuf.put(NullHandling.IS_NULL_BYTE); } else { bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putFloat(val.rhs); + bbuf.putFloat(longObjectPair.rhs); } return bbuf.array(); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index a8ba982da2e9..09d5fc4cd65a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -19,9 +19,11 @@ package org.apache.druid.query.aggregation; +import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import java.nio.ByteBuffer; +import java.util.Comparator; /** * The class serializes a Long-Long pair (SerializablePair). @@ -30,10 +32,19 @@ * The class is used on first/last Long aggregators to store the time and the first/last Long. * Long:Long -> Timestamp:Long */ -public class SerializablePairLongLongSerde extends AbstractSerializablePairSerde +public class SerializablePairLongLongSerde extends AbstractSerializableLongObjectPairSerde { public static final String TYPE_NAME = "serializablePairLongLong"; + /** + * Since SerializablePairLongFloat is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. + */ + public static Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + Long::compare, + true + ); + public SerializablePairLongLongSerde() { super(SerializablePairLongLong.class); @@ -46,7 +57,13 @@ public String getTypeName() } @Override - protected SerializablePairLongLong toPairObject(ByteBuffer buffer) + protected Comparator getLongObjectPairComparator() + { + return VALUE_COMPARATOR; + } + + @Override + protected SerializablePairLongLong toLongObjectPair(ByteBuffer buffer) { final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); long lhs = readOnlyBuffer.getLong(); @@ -59,15 +76,15 @@ protected SerializablePairLongLong toPairObject(ByteBuffer buffer) } @Override - protected byte[] pairToBytes(SerializablePairLongLong val) + protected byte[] longObjectPairToBytes(SerializablePairLongLong longObjectPair) { ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); - bbuf.putLong(val.lhs); - if (val.rhs == null) { + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { bbuf.put(NullHandling.IS_NULL_BYTE); } else { bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putLong(val.rhs); + bbuf.putLong(longObjectPair.rhs); } return bbuf.array(); } From 37dccf90051f9a3be7c01d12357a05e750b30185 Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 12 Apr 2021 11:26:12 +0800 Subject: [PATCH 14/19] fix CI Signed-off-by: frank chen --- .../query/aggregation/SerializablePairLongDoubleSerde.java | 2 +- .../query/aggregation/SerializablePairLongFloatSerde.java | 2 +- .../query/aggregation/SerializablePairLongLongSerde.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 2f8bfafc5ace..25769769793d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -40,7 +40,7 @@ public class SerializablePairLongDoubleSerde extends AbstractSerializableLongObj * Since SerializablePairLongDouble is subclass of SerializablePair, * it's safe to declare the generic type of comparator as SerializablePair. */ - public static Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + public static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( Double::compare, true ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index cc00e8c8ac36..384239047017 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -40,7 +40,7 @@ public class SerializablePairLongFloatSerde extends AbstractSerializableLongObje * Since SerializablePairLongFloat is subclass of SerializablePair, * it's safe to declare the generic type of comparator as SerializablePair. */ - public static Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + public static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( Float::compare, true ); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index 09d5fc4cd65a..6deded4af598 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -37,10 +37,10 @@ public class SerializablePairLongLongSerde extends AbstractSerializableLongObjec public static final String TYPE_NAME = "serializablePairLongLong"; /** - * Since SerializablePairLongFloat is subclass of SerializablePair, - * it's safe to declare the generic type of comparator as SerializablePair. + * Since SerializablePairLongLong is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. */ - public static Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + public static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( Long::compare, true ); From c6529b5f16fb7c3aa0f2efb91431e21862886a78 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 16 Apr 2021 09:54:40 +0800 Subject: [PATCH 15/19] Update doc Co-authored-by: Abhishek Agarwal <1477457+abhishekagarwal87@users.noreply.github.com> --- .../druid/query/aggregation/SerializablePairLongLongSerde.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index 6deded4af598..6396bde929df 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -30,7 +30,7 @@ * The serialization structure is: Long:Long *

* The class is used on first/last Long aggregators to store the time and the first/last Long. - * Long:Long -> Timestamp:Long + * (Long:timestamp, Long:value) */ public class SerializablePairLongLongSerde extends AbstractSerializableLongObjectPairSerde { From 0ac6358207f62bd829d214cb3951da65b09c1349 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 16 Apr 2021 10:26:44 +0800 Subject: [PATCH 16/19] Make subclasses implement getObjectStrategy directly --- ...stractSerializableLongObjectPairSerde.java | 55 ------------- .../SerializablePairLongDoubleSerde.java | 77 ++++++++++++------- .../SerializablePairLongFloatSerde.java | 77 ++++++++++++------- .../SerializablePairLongLongSerde.java | 75 +++++++++++------- 4 files changed, 143 insertions(+), 141 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java index e9b08ac4867c..2598d8c5bf65 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AbstractSerializableLongObjectPairSerde.java @@ -24,16 +24,13 @@ import org.apache.druid.segment.GenericColumnSerializer; import org.apache.druid.segment.column.ColumnBuilder; import org.apache.druid.segment.data.GenericIndexed; -import org.apache.druid.segment.data.ObjectStrategy; import org.apache.druid.segment.serde.ComplexColumnPartSupplier; import org.apache.druid.segment.serde.ComplexMetricExtractor; import org.apache.druid.segment.serde.ComplexMetricSerde; import org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer; import org.apache.druid.segment.writeout.SegmentWriteOutMedium; -import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.Comparator; /** * The class serializes/deserializes a Pair object for double/float/longFirst and double/float/longLast aggregators @@ -74,61 +71,9 @@ public void deserializeColumn(ByteBuffer buffer, ColumnBuilder columnBuilder) columnBuilder.setComplexColumnSupplier(new ComplexColumnPartSupplier(getTypeName(), column)); } - @Override - public ObjectStrategy getObjectStrategy() - { - return new ObjectStrategy() - { - @Override - public int compare(@Nullable T o1, @Nullable T o2) - { - return getLongObjectPairComparator().compare(o1, o2); - } - - @Override - public Class getClazz() - { - return pairClassObject; - } - - @Override - public T fromByteBuffer(ByteBuffer buffer, int numBytes) - { - return toLongObjectPair(buffer); - } - - @Override - public byte[] toBytes(@Nullable T val) - { - if (val == null) { - return new byte[]{}; - } - - return longObjectPairToBytes(val); - } - }; - } - @Override public GenericColumnSerializer getSerializer(SegmentWriteOutMedium segmentWriteOutMedium, String column) { return LargeColumnSupportedComplexColumnSerializer.create(segmentWriteOutMedium, column, this.getObjectStrategy()); } - - /** - * get comparator for Pair object - * It's recommended to use {@link SerializablePair#createNullHandlingComparator(Comparator, boolean)} to instance a concret comparator. - * - */ - protected abstract Comparator getLongObjectPairComparator(); - - /** - * deserialize a Pair object from buffer - */ - protected abstract T toLongObjectPair(ByteBuffer buffer); - - /** - * serialize a Pair object to byte array - */ - protected abstract byte[] longObjectPairToBytes(T longObjectPair); } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 25769769793d..245f34f61ee0 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -21,7 +21,9 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.data.ObjectStrategy; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Comparator; @@ -30,7 +32,7 @@ * The serialization structure is: Long:Double *

* The class is used on first/last Double aggregators to store the time and the first/last Double. - * Long:Double -> Timestamp:Double + * (Long:timestamp, Float:value) */ public class SerializablePairLongDoubleSerde extends AbstractSerializableLongObjectPairSerde { @@ -40,7 +42,7 @@ public class SerializablePairLongDoubleSerde extends AbstractSerializableLongObj * Since SerializablePairLongDouble is subclass of SerializablePair, * it's safe to declare the generic type of comparator as SerializablePair. */ - public static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + private static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( Double::compare, true ); @@ -57,35 +59,52 @@ public String getTypeName() } @Override - protected Comparator getLongObjectPairComparator() + public ObjectStrategy getObjectStrategy() { - return VALUE_COMPARATOR; - } + return new ObjectStrategy() + { + @Override + public int compare(@Nullable SerializablePairLongDouble o1, @Nullable SerializablePairLongDouble o2) + { + return VALUE_COMPARATOR.compare(o1, o2); + } - @Override - protected SerializablePairLongDouble toLongObjectPair(ByteBuffer buffer) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); - } else { - return new SerializablePairLongDouble(lhs, null); - } - } + @Override + public Class getClazz() + { + return SerializablePairLongDouble.class; + } - @Override - protected byte[] longObjectPairToBytes(SerializablePairLongDouble longObjectPair) - { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); - bbuf.putLong(longObjectPair.lhs); - if (longObjectPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putDouble(longObjectPair.rhs); - } - return bbuf.array(); + @Override + public SerializablePairLongDouble fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); + } else { + return new SerializablePairLongDouble(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongDouble longObjectPair) + { + if (longObjectPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putDouble(longObjectPair.rhs); + } + return bbuf.array(); + } + }; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index 384239047017..33c703a58a5d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -21,7 +21,9 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.data.ObjectStrategy; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Comparator; @@ -30,7 +32,7 @@ * The serialization structure is: Long:Float *

* The class is used on first/last Float aggregators to store the time and the first/last Float. - * Long:Float -> Timestamp:Float + * (Long:timestamp, Float:value) */ public class SerializablePairLongFloatSerde extends AbstractSerializableLongObjectPairSerde { @@ -40,7 +42,7 @@ public class SerializablePairLongFloatSerde extends AbstractSerializableLongObje * Since SerializablePairLongFloat is subclass of SerializablePair, * it's safe to declare the generic type of comparator as SerializablePair. */ - public static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + private static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( Float::compare, true ); @@ -57,35 +59,52 @@ public String getTypeName() } @Override - protected Comparator getLongObjectPairComparator() + public ObjectStrategy getObjectStrategy() { - return VALUE_COMPARATOR; - } + return new ObjectStrategy() + { + @Override + public int compare(@Nullable SerializablePairLongFloat o1, @Nullable SerializablePairLongFloat o2) + { + return VALUE_COMPARATOR.compare(o1, o2); + } - @Override - protected SerializablePairLongFloat toLongObjectPair(ByteBuffer buffer) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); - } else { - return new SerializablePairLongFloat(lhs, null); - } - } + @Override + public Class getClazz() + { + return SerializablePairLongFloat.class; + } - @Override - protected byte[] longObjectPairToBytes(SerializablePairLongFloat longObjectPair) - { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); - bbuf.putLong(longObjectPair.lhs); - if (longObjectPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putFloat(longObjectPair.rhs); - } - return bbuf.array(); + @Override + public SerializablePairLongFloat fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); + } else { + return new SerializablePairLongFloat(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongFloat longObjectPair) + { + if (longObjectPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putFloat(longObjectPair.rhs); + } + return bbuf.array(); + } + }; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index 6396bde929df..a90755d6c2dc 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -21,7 +21,9 @@ import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.segment.data.ObjectStrategy; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.Comparator; @@ -40,7 +42,7 @@ public class SerializablePairLongLongSerde extends AbstractSerializableLongObjec * Since SerializablePairLongLong is subclass of SerializablePair, * it's safe to declare the generic type of comparator as SerializablePair. */ - public static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( + private static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( Long::compare, true ); @@ -57,35 +59,52 @@ public String getTypeName() } @Override - protected Comparator getLongObjectPairComparator() + public ObjectStrategy getObjectStrategy() { - return VALUE_COMPARATOR; - } + return new ObjectStrategy() + { + @Override + public int compare(@Nullable SerializablePairLongLong o1, @Nullable SerializablePairLongLong o2) + { + return VALUE_COMPARATOR.compare(o1, o2); + } - @Override - protected SerializablePairLongLong toLongObjectPair(ByteBuffer buffer) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); - } else { - return new SerializablePairLongLong(lhs, null); - } - } + @Override + public Class getClazz() + { + return SerializablePairLongLong.class; + } - @Override - protected byte[] longObjectPairToBytes(SerializablePairLongLong longObjectPair) - { - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); - bbuf.putLong(longObjectPair.lhs); - if (longObjectPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putLong(longObjectPair.rhs); - } - return bbuf.array(); + @Override + public SerializablePairLongLong fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); + } else { + return new SerializablePairLongLong(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongLong longObjectPair) + { + if (longObjectPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putLong(longObjectPair.rhs); + } + return bbuf.array(); + } + }; } } From 778550788d58824c564abcd670bdf1e981458a64 Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 19 Apr 2021 17:31:11 +0800 Subject: [PATCH 17/19] Address comments --- .../SerializablePairLongDoubleSerde.java | 112 +++++++++--------- .../SerializablePairLongFloatSerde.java | 110 ++++++++--------- .../SerializablePairLongLongSerde.java | 110 ++++++++--------- 3 files changed, 172 insertions(+), 160 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java index 245f34f61ee0..270c5ecd140d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongDoubleSerde.java @@ -32,20 +32,68 @@ * The serialization structure is: Long:Double *

* The class is used on first/last Double aggregators to store the time and the first/last Double. - * (Long:timestamp, Float:value) + * (Long:timestamp, Double:value) */ public class SerializablePairLongDoubleSerde extends AbstractSerializableLongObjectPairSerde { public static final String TYPE_NAME = "serializablePairLongDouble"; - /** - * Since SerializablePairLongDouble is subclass of SerializablePair, - * it's safe to declare the generic type of comparator as SerializablePair. - */ - private static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( - Double::compare, - true - ); + private static class ObjectStrategyImpl implements ObjectStrategy + { + /** + * Since SerializablePairLongDouble is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. + */ + private final Comparator> pairComparator = SerializablePair.createNullHandlingComparator( + Double::compare, + true + ); + + @Override + public int compare(@Nullable SerializablePairLongDouble o1, @Nullable SerializablePairLongDouble o2) + { + return pairComparator.compare(o1, o2); + } + + @Override + public Class getClazz() + { + return SerializablePairLongDouble.class; + } + + @Override + public SerializablePairLongDouble fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); + } else { + return new SerializablePairLongDouble(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongDouble longObjectPair) + { + if (longObjectPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putDouble(longObjectPair.rhs); + } + return bbuf.array(); + } + } + + private static final ObjectStrategy OBJECT_STRATEGY = new ObjectStrategyImpl(); public SerializablePairLongDoubleSerde() { @@ -61,50 +109,6 @@ public String getTypeName() @Override public ObjectStrategy getObjectStrategy() { - return new ObjectStrategy() - { - @Override - public int compare(@Nullable SerializablePairLongDouble o1, @Nullable SerializablePairLongDouble o2) - { - return VALUE_COMPARATOR.compare(o1, o2); - } - - @Override - public Class getClazz() - { - return SerializablePairLongDouble.class; - } - - @Override - public SerializablePairLongDouble fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongDouble(lhs, readOnlyBuffer.getDouble()); - } else { - return new SerializablePairLongDouble(lhs, null); - } - } - - @Override - public byte[] toBytes(@Nullable SerializablePairLongDouble longObjectPair) - { - if (longObjectPair == null) { - return new byte[]{}; - } - - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Double.BYTES); - bbuf.putLong(longObjectPair.lhs); - if (longObjectPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putDouble(longObjectPair.rhs); - } - return bbuf.array(); - } - }; + return OBJECT_STRATEGY; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java index 33c703a58a5d..db4ae83246cf 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongFloatSerde.java @@ -38,14 +38,62 @@ public class SerializablePairLongFloatSerde extends AbstractSerializableLongObje { public static final String TYPE_NAME = "serializablePairLongFloat"; - /** - * Since SerializablePairLongFloat is subclass of SerializablePair, - * it's safe to declare the generic type of comparator as SerializablePair. - */ - private static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( - Float::compare, - true - ); + private static class ObjectStrategyImpl implements ObjectStrategy + { + /** + * Since SerializablePairLongFloat is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. + */ + private final Comparator> pairComparator = SerializablePair.createNullHandlingComparator( + Float::compare, + true + ); + + @Override + public int compare(@Nullable SerializablePairLongFloat o1, @Nullable SerializablePairLongFloat o2) + { + return pairComparator.compare(o1, o2); + } + + @Override + public Class getClazz() + { + return SerializablePairLongFloat.class; + } + + @Override + public SerializablePairLongFloat fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); + } else { + return new SerializablePairLongFloat(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongFloat longObjectPair) + { + if (longObjectPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putFloat(longObjectPair.rhs); + } + return bbuf.array(); + } + } + + private static final ObjectStrategy OBJECT_STRATEGY = new ObjectStrategyImpl(); public SerializablePairLongFloatSerde() { @@ -61,50 +109,6 @@ public String getTypeName() @Override public ObjectStrategy getObjectStrategy() { - return new ObjectStrategy() - { - @Override - public int compare(@Nullable SerializablePairLongFloat o1, @Nullable SerializablePairLongFloat o2) - { - return VALUE_COMPARATOR.compare(o1, o2); - } - - @Override - public Class getClazz() - { - return SerializablePairLongFloat.class; - } - - @Override - public SerializablePairLongFloat fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongFloat(lhs, readOnlyBuffer.getFloat()); - } else { - return new SerializablePairLongFloat(lhs, null); - } - } - - @Override - public byte[] toBytes(@Nullable SerializablePairLongFloat longObjectPair) - { - if (longObjectPair == null) { - return new byte[]{}; - } - - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Float.BYTES); - bbuf.putLong(longObjectPair.lhs); - if (longObjectPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putFloat(longObjectPair.rhs); - } - return bbuf.array(); - } - }; + return OBJECT_STRATEGY; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java index a90755d6c2dc..76208f10dd23 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SerializablePairLongLongSerde.java @@ -38,14 +38,62 @@ public class SerializablePairLongLongSerde extends AbstractSerializableLongObjec { public static final String TYPE_NAME = "serializablePairLongLong"; - /** - * Since SerializablePairLongLong is subclass of SerializablePair, - * it's safe to declare the generic type of comparator as SerializablePair. - */ - private static final Comparator> VALUE_COMPARATOR = SerializablePair.createNullHandlingComparator( - Long::compare, - true - ); + private static class ObjectStrategyImpl implements ObjectStrategy + { + /** + * Since SerializablePairLongLong is subclass of SerializablePair, + * it's safe to declare the generic type of comparator as SerializablePair. + */ + private final Comparator> pairComparator = SerializablePair.createNullHandlingComparator( + Long::compare, + true + ); + + @Override + public int compare(@Nullable SerializablePairLongLong o1, @Nullable SerializablePairLongLong o2) + { + return pairComparator.compare(o1, o2); + } + + @Override + public Class getClazz() + { + return SerializablePairLongLong.class; + } + + @Override + public SerializablePairLongLong fromByteBuffer(ByteBuffer buffer, int numBytes) + { + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); + long lhs = readOnlyBuffer.getLong(); + boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; + if (isNotNull) { + return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); + } else { + return new SerializablePairLongLong(lhs, null); + } + } + + @Override + public byte[] toBytes(@Nullable SerializablePairLongLong longObjectPair) + { + if (longObjectPair == null) { + return new byte[]{}; + } + + ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); + bbuf.putLong(longObjectPair.lhs); + if (longObjectPair.rhs == null) { + bbuf.put(NullHandling.IS_NULL_BYTE); + } else { + bbuf.put(NullHandling.IS_NOT_NULL_BYTE); + bbuf.putLong(longObjectPair.rhs); + } + return bbuf.array(); + } + } + + private static final ObjectStrategy OBJECT_STRATEGY = new ObjectStrategyImpl(); public SerializablePairLongLongSerde() { @@ -61,50 +109,6 @@ public String getTypeName() @Override public ObjectStrategy getObjectStrategy() { - return new ObjectStrategy() - { - @Override - public int compare(@Nullable SerializablePairLongLong o1, @Nullable SerializablePairLongLong o2) - { - return VALUE_COMPARATOR.compare(o1, o2); - } - - @Override - public Class getClazz() - { - return SerializablePairLongLong.class; - } - - @Override - public SerializablePairLongLong fromByteBuffer(ByteBuffer buffer, int numBytes) - { - final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); - long lhs = readOnlyBuffer.getLong(); - boolean isNotNull = readOnlyBuffer.get() == NullHandling.IS_NOT_NULL_BYTE; - if (isNotNull) { - return new SerializablePairLongLong(lhs, readOnlyBuffer.getLong()); - } else { - return new SerializablePairLongLong(lhs, null); - } - } - - @Override - public byte[] toBytes(@Nullable SerializablePairLongLong longObjectPair) - { - if (longObjectPair == null) { - return new byte[]{}; - } - - ByteBuffer bbuf = ByteBuffer.allocate(Long.BYTES + Byte.BYTES + Long.BYTES); - bbuf.putLong(longObjectPair.lhs); - if (longObjectPair.rhs == null) { - bbuf.put(NullHandling.IS_NULL_BYTE); - } else { - bbuf.put(NullHandling.IS_NOT_NULL_BYTE); - bbuf.putLong(longObjectPair.rhs); - } - return bbuf.array(); - } - }; + return OBJECT_STRATEGY; } } From 901097b730d7c4d09b4ae075893b5434795df86c Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 23 Apr 2021 14:07:21 +0800 Subject: [PATCH 18/19] Add null related test --- .../first/DoubleFirstAggregator.java | 6 +- .../first/FloatFirstAggregator.java | 6 +- .../first/LongFirstAggregator.java | 6 +- .../first/NumericFirstAggregator.java | 18 +-- .../last/DoubleLastAggregator.java | 6 +- .../aggregation/last/FloatLastAggregator.java | 6 +- .../last/GenericLastAggregateCombiner.java | 2 +- .../aggregation/last/LongLastAggregator.java | 9 +- .../last/NumericLastAggregator.java | 18 +-- .../first/FloatFirstAggregationTest.java | 111 +++++++++++++++--- .../last/FloatLastAggregationTest.java | 79 +++++++++++-- .../druid/segment/IndexMergerRollupTest.java | 89 +++++++++++--- 12 files changed, 273 insertions(+), 83 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java index 0942c375e837..a0148c48f496 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregator.java @@ -36,15 +36,15 @@ public DoubleFirstAggregator(BaseLongColumnValueSelector timeSelector, } @Override - void setCurrentValue(ColumnValueSelector valueSelector) + void setFirstValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getDouble(); } @Override - void setCurrentValue(Number number) + void setFirstValue(Number firstValue) { - firstValue = number.doubleValue(); + this.firstValue = firstValue.doubleValue(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java index bbf568f12470..d02bfdb922cd 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FloatFirstAggregator.java @@ -38,15 +38,15 @@ public FloatFirstAggregator( } @Override - void setCurrentValue(ColumnValueSelector valueSelector) + void setFirstValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getFloat(); } @Override - void setCurrentValue(Number number) + void setFirstValue(Number firstValue) { - firstValue = number.floatValue(); + this.firstValue = firstValue.floatValue(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java index b76b67bbae0e..53559674393c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/LongFirstAggregator.java @@ -36,15 +36,15 @@ public LongFirstAggregator(BaseLongColumnValueSelector timeSelector, } @Override - void setCurrentValue(ColumnValueSelector valueSelector) + void setFirstValue(ColumnValueSelector valueSelector) { firstValue = valueSelector.getLong(); } @Override - void setCurrentValue(Number number) + void setFirstValue(Number firstValue) { - firstValue = number.longValue(); + this.firstValue = firstValue.longValue(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java index 444bcc39bb46..3d4d90090b06 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstAggregator.java @@ -30,7 +30,7 @@ */ public abstract class NumericFirstAggregator implements Aggregator { - private final boolean useDefault = NullHandling.replaceWithDefault(); + private static final boolean USE_DEFAULT = NullHandling.replaceWithDefault(); private final BaseLongColumnValueSelector timeSelector; private final ColumnValueSelector valueSelector; private final boolean needsFoldCheck; @@ -49,15 +49,18 @@ public NumericFirstAggregator( this.needsFoldCheck = needsFoldCheck; firstTime = Long.MAX_VALUE; - rhsNull = !useDefault; + rhsNull = !USE_DEFAULT; } /** * Store the current primitive typed 'first' value */ - abstract void setCurrentValue(ColumnValueSelector valueSelector); + abstract void setFirstValue(ColumnValueSelector valueSelector); - abstract void setCurrentValue(Number number); + /** + * Store a non-null first value + */ + abstract void setFirstValue(Number firstValue); @Override public void aggregate() @@ -80,7 +83,7 @@ public void aggregate() rhsNull = true; } else { rhsNull = false; - setCurrentValue(pair.rhs); + setFirstValue(pair.rhs); } } @@ -91,10 +94,11 @@ public void aggregate() long time = timeSelector.getLong(); if (time < firstTime) { firstTime = time; - if (useDefault || !valueSelector.isNull()) { - setCurrentValue(valueSelector); + if (USE_DEFAULT || !valueSelector.isNull()) { + setFirstValue(valueSelector); rhsNull = false; } else { + setFirstValue(0); rhsNull = true; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java index 504e113a8597..e19479b21e70 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastAggregator.java @@ -36,15 +36,15 @@ public DoubleLastAggregator(BaseLongColumnValueSelector timeSelector, } @Override - void setCurrentValue(ColumnValueSelector valueSelector) + void setLastValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getDouble(); } @Override - void setCurrentValue(Number number) + void setLastValue(Number lastValue) { - lastValue = number.doubleValue(); + this.lastValue = lastValue.doubleValue(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java index 11a19a56f71f..b874cf1ac79e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastAggregator.java @@ -36,15 +36,15 @@ public FloatLastAggregator(BaseLongColumnValueSelector timeSelector, } @Override - void setCurrentValue(ColumnValueSelector valueSelector) + void setLastValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getFloat(); } @Override - void setCurrentValue(Number number) + void setLastValue(Number lastValue) { - lastValue = number.floatValue(); + this.lastValue = lastValue.floatValue(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java index 5fb2f3960fe0..308b37308111 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/GenericLastAggregateCombiner.java @@ -49,7 +49,7 @@ public void fold(ColumnValueSelector selector) { T newValue = (T) selector.getObject(); - if (Longs.compare(lastValue.lhs, newValue.lhs) < 0) { + if (Longs.compare(lastValue.lhs, newValue.lhs) <= 0) { lastValue = newValue; } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java index 598cd473013b..b9aaa358c606 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregator.java @@ -29,23 +29,22 @@ public class LongLastAggregator extends NumericLastAggregator public LongLastAggregator(BaseLongColumnValueSelector timeSelector, ColumnValueSelector valueSelector, - boolean needsFoldCheck - ) + boolean needsFoldCheck) { super(timeSelector, valueSelector, needsFoldCheck); lastValue = 0; } @Override - void setCurrentValue(ColumnValueSelector valueSelector) + void setLastValue(ColumnValueSelector valueSelector) { lastValue = valueSelector.getLong(); } @Override - void setCurrentValue(Number number) + void setLastValue(Number lastValue) { - lastValue = number.longValue(); + this.lastValue = lastValue.longValue(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java index 220da0691372..9f25eea71cab 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastAggregator.java @@ -32,10 +32,11 @@ */ public abstract class NumericLastAggregator implements Aggregator { - private final boolean useDefault = NullHandling.replaceWithDefault(); + private static final boolean USE_DEFAULT = NullHandling.replaceWithDefault(); private final BaseLongColumnValueSelector timeSelector; private final boolean needsFoldCheck; private final ColumnValueSelector valueSelector; + long lastTime; boolean rhsNull; @@ -50,14 +51,14 @@ public NumericLastAggregator( this.needsFoldCheck = needsFoldCheck; lastTime = Long.MIN_VALUE; - rhsNull = !useDefault; + rhsNull = !USE_DEFAULT; } @Override public void aggregate() { if (needsFoldCheck) { - // Need to read this first (before time), just in case it's a SerializablePairLongString (we don't know; it's + // Need to read this first (before time), just in case it's a SerializablePair (we don't know; it's // detected at query time). final Object object = valueSelector.getObject(); @@ -73,7 +74,7 @@ public void aggregate() rhsNull = true; } else { rhsNull = false; - setCurrentValue(pair.rhs); + setLastValue(pair.rhs); } } return; @@ -83,10 +84,11 @@ public void aggregate() long time = timeSelector.getLong(); if (time >= lastTime) { lastTime = time; - if (useDefault || !valueSelector.isNull()) { - setCurrentValue(valueSelector); + if (USE_DEFAULT || !valueSelector.isNull()) { + setLastValue(valueSelector); rhsNull = false; } else { + setLastValue(0); rhsNull = true; } } @@ -101,7 +103,7 @@ public void close() /** * Store the current primitive typed 'first' value */ - abstract void setCurrentValue(ColumnValueSelector valueSelector); + abstract void setLastValue(ColumnValueSelector valueSelector); - abstract void setCurrentValue(Number number); + abstract void setLastValue(Number lastValue); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java index f68b1a8e016e..fbe3d6c230ed 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java @@ -54,10 +54,13 @@ public class FloatFirstAggregationTest extends InitializedNullHandlingTest private float[] floats = {1.1f, 2.7f, 3.5f, 1.3f}; private long[] times = {12, 10, 5344, 7899999}; private SerializablePair[] pairs = { + new SerializablePair<>(1567899920L, null), new SerializablePair<>(1467225096L, 134.3f), new SerializablePair<>(23163L, 1232.212f), new SerializablePair<>(742L, 18f), - new SerializablePair<>(111111L, 233.5232f) + new SerializablePair<>(111111L, 233.5232f), + new SerializablePair<>(500L, null), + new SerializablePair<>(500L, 400.f) }; @Before @@ -74,6 +77,9 @@ public void setup() EasyMock.expect(colSelectorFactory.makeColumnValueSelector("billy")).andReturn(objectSelector).atLeastOnce(); } + /** + * test aggregator on value selector column + */ @Test public void testDoubleFirstAggregator() { @@ -96,6 +102,9 @@ public void testDoubleFirstAggregator() Assert.assertEquals(floats[1], agg.getFloat(), 0.0001); } + /** + * test aggregator on value selector column + */ @Test public void testDoubleFirstBufferAggregator() { @@ -142,6 +151,9 @@ public void testComparatorWithNulls() Assert.assertEquals(-1, comparator.compare(pair2, pair1)); } + /** + * test aggregator on an object selector column + */ @Test public void testDoubleFirstCombiningAggregator() { @@ -151,20 +163,52 @@ public void testDoubleFirstCombiningAggregator() Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); + // + // aggregate first 5 events, pair[3] is supposed to be the first + // aggregate(agg); aggregate(agg); aggregate(agg); aggregate(agg); + agg.aggregate(); + objectSelector.increment(); Pair result = (Pair) agg.get(); - Pair expected = (Pair) pairs[2]; + Pair expected = (Pair) pairs[3]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs, 0.0001); Assert.assertEquals(expected.rhs.longValue(), agg.getLong()); Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); + + // + // aggregator once more, pair[5] will be the first + // + agg.aggregate(); + objectSelector.increment(); + result = (Pair) agg.get(); + expected = (Pair) pairs[5]; + + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertNull(result.rhs); + + // + // aggregate once more, pair[6] has the same timestamp as pair[5], but it won't be the first + // + agg.aggregate(); + objectSelector.increment(); + result = (Pair) agg.get(); + expected = (Pair) pairs[5]; + + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertNull(result.rhs); } + /** + * test aggregator on an object column + */ @Test public void testDoubleFirstCombiningBufferAggregator() { @@ -172,26 +216,52 @@ public void testDoubleFirstCombiningBufferAggregator() ValueType.COMPLEX)); EasyMock.replay(colSelectorFactory); - BufferAggregator agg = combiningAggFactory.factorizeBuffered( - colSelectorFactory); + BufferAggregator agg = combiningAggFactory.factorizeBuffered(colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[floatFirstAggregatorFactory.getMaxIntermediateSizeWithNulls()]); agg.init(buffer, 0); + // + // aggregate first 5 events, pair[3] is the first + // + aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); Pair result = (Pair) agg.get(buffer, 0); - Pair expected = (Pair) pairs[2]; + Pair expected = (Pair) pairs[3]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs, 0.0001); Assert.assertEquals(expected.rhs.longValue(), agg.getLong(buffer, 0)); Assert.assertEquals(expected.rhs, agg.getFloat(buffer, 0), 0.0001); - } + // + // aggregate once more, pair[5] is the first + // + agg.aggregate(buffer, 0); + objectSelector.increment(); + result = (Pair) agg.get(buffer, 0); + expected = (Pair) pairs[5]; + + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertNull(result.rhs); + + // + // aggregate once more, pair[6] has the same timestamp as pair[5], but it won't be the first + // + agg.aggregate(buffer, 0); + objectSelector.increment(); + result = (Pair) agg.get(buffer, 0); + expected = (Pair) pairs[5]; + + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertNull(result.rhs); + } @Test public void testSerde() throws Exception @@ -207,32 +277,41 @@ public void testFloatFirstAggregateCombiner() AggregateCombiner floatFirstAggregateCombiner = combiningAggFactory.makeAggregateCombiner(); SerializablePair[] inputPairs = { - new SerializablePair<>(5L, 134.3f), - new SerializablePair<>(4L, 1232.212f), - new SerializablePair<>(3L, 18f), - new SerializablePair<>(6L, 233.5232f) + new SerializablePair<>(6L, null), + new SerializablePair<>(6L, 134.3f), + new SerializablePair<>(5L, 1232.212f), + new SerializablePair<>(4L, 18f), + new SerializablePair<>(7L, 233.5232f), + new SerializablePair<>(0L, null) }; TestObjectColumnSelector columnSelector = new TestObjectColumnSelector<>(inputPairs); floatFirstAggregateCombiner.reset(columnSelector); Assert.assertEquals(inputPairs[0], floatFirstAggregateCombiner.getObject()); - // inputPairs[1] has lower time value, it should be the first + // inputPairs[1].first > inputPair[0].first, it should NOT be the first columnSelector.increment(); floatFirstAggregateCombiner.fold(columnSelector); - Assert.assertEquals(inputPairs[1], floatFirstAggregateCombiner.getObject()); + Assert.assertEquals(inputPairs[0], floatFirstAggregateCombiner.getObject()); - // inputPairs[2] has lower time value, it should be the first + // inputPairs[2].first < inputPair[0].first, it should be the first columnSelector.increment(); floatFirstAggregateCombiner.fold(columnSelector); Assert.assertEquals(inputPairs[2], floatFirstAggregateCombiner.getObject()); - // inputPairs[3] has the max time value, it should NOT be the first + // inputPairs[3].first is the lowest, it should be the first columnSelector.increment(); floatFirstAggregateCombiner.fold(columnSelector); - Assert.assertEquals(inputPairs[2], floatFirstAggregateCombiner.getObject()); + Assert.assertEquals(inputPairs[3], floatFirstAggregateCombiner.getObject()); - floatFirstAggregateCombiner.reset(columnSelector); + // inputPairs[4] is not the lowest, it should NOT be the first + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); Assert.assertEquals(inputPairs[3], floatFirstAggregateCombiner.getObject()); + + columnSelector.increment(); + floatFirstAggregateCombiner.fold(columnSelector); + + Assert.assertEquals(inputPairs[5], floatFirstAggregateCombiner.getObject()); } private void aggregate( diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java index b5d4e11e6967..8f349afa340d 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java @@ -54,10 +54,13 @@ public class FloatLastAggregationTest extends InitializedNullHandlingTest private float[] floats = {1.1897f, 0.001f, 86.23f, 166.228f}; private long[] times = {8224, 6879, 2436, 7888}; private SerializablePair[] pairs = { + new SerializablePair<>(111L, null), new SerializablePair<>(52782L, 134.3f), new SerializablePair<>(65492L, 1232.212f), new SerializablePair<>(69134L, 18.1233f), - new SerializablePair<>(11111L, 233.5232f) + new SerializablePair<>(11111L, 233.5232f), + new SerializablePair<>(99999L, 99999.f), + new SerializablePair<>(99999L, null) }; @Before @@ -93,6 +96,8 @@ public void testDoubleLastAggregator() Assert.assertEquals(floats[0], result.rhs, 0.0001); Assert.assertEquals((long) floats[0], agg.getLong()); Assert.assertEquals(floats[0], agg.getFloat(), 0.0001); + + } @Test @@ -149,18 +154,43 @@ public void testDoubleLastCombiningAggregator() Aggregator agg = combiningAggFactory.factorize(colSelectorFactory); - aggregate(agg); - aggregate(agg); - aggregate(agg); - aggregate(agg); + agg.aggregate(); + objectSelector.increment(); + agg.aggregate(); + objectSelector.increment(); + agg.aggregate(); + objectSelector.increment(); + agg.aggregate(); + objectSelector.increment(); + agg.aggregate(); + objectSelector.increment(); Pair result = (Pair) agg.get(); - Pair expected = (Pair) pairs[2]; + // pair[3] has the largest timestamp in first 5 events + Pair expected = (Pair) pairs[3]; + + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(expected.rhs, result.rhs, 0.0001); + Assert.assertEquals(expected.rhs.longValue(), agg.getLong()); + Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); + // aggregate once more, the last will change to pair[5] event + agg.aggregate(); + objectSelector.increment(); + result = (Pair) agg.get(); + expected = (Pair) pairs[5]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs, 0.0001); Assert.assertEquals(expected.rhs.longValue(), agg.getLong()); Assert.assertEquals(expected.rhs, agg.getFloat(), 0.0001); + + // aggregate once more, now the last event has the same timestamp as the last-1 event, it will be the last + agg.aggregate(); + objectSelector.increment(); + result = (Pair) agg.get(); + expected = (Pair) pairs[6]; + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(result.rhs, null); } @Test @@ -176,20 +206,44 @@ public void testDoubleLastCombiningBufferAggregator() ByteBuffer buffer = ByteBuffer.wrap(new byte[floatLastAggregatorFactory.getMaxIntermediateSizeWithNulls()]); agg.init(buffer, 0); - aggregate(agg, buffer, 0); - aggregate(agg, buffer, 0); - aggregate(agg, buffer, 0); - aggregate(agg, buffer, 0); + // aggregate first 5 events, pair[3] is the last + agg.aggregate(buffer, 0); + objectSelector.increment(); + agg.aggregate(buffer, 0); + objectSelector.increment(); + agg.aggregate(buffer, 0); + objectSelector.increment(); + agg.aggregate(buffer, 0); + objectSelector.increment(); + agg.aggregate(buffer, 0); + objectSelector.increment(); Pair result = (Pair) agg.get(buffer, 0); - Pair expected = (Pair) pairs[2]; + Pair expected = (Pair) pairs[3]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs, 0.0001); Assert.assertEquals(expected.rhs.longValue(), agg.getLong(buffer, 0)); Assert.assertEquals(expected.rhs, agg.getFloat(buffer, 0), 0.0001); - } + // aggregate once more, pair[5] is the last + agg.aggregate(buffer, 0); + objectSelector.increment(); + result = (Pair) agg.get(buffer, 0); + expected = (Pair) pairs[5]; + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(expected.rhs, result.rhs, 0.0001); + Assert.assertEquals(expected.rhs.longValue(), agg.getLong(buffer, 0)); + Assert.assertEquals(expected.rhs, agg.getFloat(buffer, 0), 0.0001); + + // aggregate once more, pair[6] has the same timestamp with pair[5], it will be the last + agg.aggregate(buffer, 0); + objectSelector.increment(); + result = (Pair) agg.get(buffer, 0); + expected = (Pair) pairs[5]; + Assert.assertEquals(expected.lhs, result.lhs); + Assert.assertEquals(result.rhs, null); + } @Test public void testSerde() throws Exception @@ -233,6 +287,7 @@ public void testDoubleLastAggregateCombiner() Assert.assertEquals(inputPairs[3], floatLastAggregateCombiner.getObject()); } + private void aggregate( Aggregator agg ) diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index 0d77321fee42..1d20d6225415 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -21,8 +21,10 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; @@ -87,7 +89,7 @@ public void setUp() indexSpec = new IndexSpec(); } - private void testFirstLastRollup( + private QueryableIndex testFirstLastRollup( final List> eventsList, final List dimensions, final AggregatorFactory... aggregatorFactories @@ -96,6 +98,8 @@ private void testFirstLastRollup( final File tempDir = temporaryFolder.newFolder(); List indexes = new ArrayList<>(); + + // set timestamp of all events to the same so that rollup will be applied to these events Instant time = Instant.now(); for (Map events : eventsList) { @@ -107,88 +111,135 @@ private void testFirstLastRollup( File indexFile = indexMerger .mergeQueryableIndex(indexes, true, aggregatorFactories, tempDir, indexSpec, null, -1); - try (QueryableIndex mergedIndex = indexIO.loadIndex(indexFile)) { - Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); - } + return indexIO.loadIndex(indexFile); } @Test public void testStringFirstRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( strEventsList, ImmutableList.of("d"), new StringFirstAggregatorFactory("m", "m", 1024) - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testStringLastRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( strEventsList, ImmutableList.of("d"), new StringLastAggregatorFactory("m", "m", 1024) - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testDoubleFirstRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( doubleEventsList, ImmutableList.of("d"), new DoubleFirstAggregatorFactory("m", "m") - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testDoubleLastRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( doubleEventsList, ImmutableList.of("d"), new DoubleLastAggregatorFactory("m", "m") - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testFloatFirstRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( floatEventsList, ImmutableList.of("d"), new FloatFirstAggregatorFactory("m", "m") - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testFloatLastRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( floatEventsList, ImmutableList.of("d"), new FloatLastAggregatorFactory("m", "m") - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testLongFirstRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( longEventsList, ImmutableList.of("d"), new LongFirstAggregatorFactory("m", "m") - ); + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } } @Test public void testLongLastRollup() throws Exception { - testFirstLastRollup( + try (QueryableIndex mergedIndex = testFirstLastRollup( longEventsList, ImmutableList.of("d"), new LongLastAggregatorFactory("m", "m") + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + } + } + + @Test + public void testLongFirstRollupWithNull() throws Exception + { + final List> longEventsList = Arrays.asList( + // m == null + ImmutableMap.of("d", "d1"), + + ImmutableMap.of("d", "d1", "m", 1L), + ImmutableMap.of("d", "d1", "m", 2L) ); + + try (QueryableIndex mergedIndex = testFirstLastRollup( + longEventsList, + ImmutableList.of("d"), + new LongFirstAggregatorFactory("m", "m") + )) { + Assert.assertEquals("Number of rows should be 1", 1, mergedIndex.getNumRows()); + + Object o = mergedIndex.getColumnHolder("m") + .getColumn() + .makeColumnValueSelector(new SimpleAscendingOffset(0)) + .getObject(); + Assert.assertEquals(o.getClass(), SerializablePairLongLong.class); + + // since input events have the same timestamp, longFirst aggregator should return the first event + if (NullHandling.replaceWithDefault()) { + Assert.assertEquals(0L, ((SerializablePairLongLong) o).rhs.longValue()); + } else { + Assert.assertEquals(null, ((SerializablePairLongLong) o).rhs); + } + } } } From 6e94ce83729780c296022d461c340a15678524dc Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 23 Apr 2021 14:07:33 +0800 Subject: [PATCH 19/19] Update doc --- docs/querying/aggregations.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/querying/aggregations.md b/docs/querying/aggregations.md index a25482b8bc30..ff73b5c65e20 100644 --- a/docs/querying/aggregations.md +++ b/docs/querying/aggregations.md @@ -134,8 +134,6 @@ Computes and returns arithmetic mean of a column values as 64 bit float value. T ### First / Last aggregator -(Double/Float/Long) First and Last aggregator cannot be used in ingestion spec, and should only be specified as part of queries. - Note that queries with first/last aggregators on a segment created with rollup enabled will return the rolled up value, and not the last value within the raw ingested data. #### `doubleFirst` aggregator