From 2c24820d5046aff2e85f06b01439e4421ea0758b Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 3 Apr 2024 11:26:08 +0530 Subject: [PATCH 01/10] init, add stuff for double, last --- .../aggregation/first/FirstLastUtils.java | 65 ++++ .../last/DoubleLastVectorAggregator.java | 46 ++- .../last/FloatLastVectorAggregator.java | 53 +++- .../last/NumericLastVectorAggregator.java | 299 +++++++++++------- ...leStringLastDimensionVectorAggregator.java | 6 +- 5 files changed, 326 insertions(+), 143 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java index b9ef88dfdcf9..18f9e78c91ab 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -19,10 +19,15 @@ package org.apache.druid.query.aggregation.first; +import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.BaseObjectColumnValueSelector; +import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -66,4 +71,64 @@ public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) return clazz.isAssignableFrom(pairClass) || pairClass.isAssignableFrom(clazz); } + + @Nullable + public static SerializablePairLongDouble readDoublePairFromVectorSelectors( + @Nullable boolean[] timeNullityVector, + long[] timeVector, + Object[] objectVector, + int index + ) + { + final long time; + final Double value; + + final Object object = objectVector[index]; + + if (object instanceof SerializablePairLongDouble) { + // We got a folded object, ignore timeSelector completely, the object has all the info it requires + final SerializablePairLongDouble pair = (SerializablePairLongDouble) object; + // if time == null, don't aggregate + if (pair.lhs == null) { + return null; + } + time = pair.lhs; + value = pair.rhs; + } else { + if (timeNullityVector != null && timeNullityVector[index]) { + // Donot aggregate pairs where time is unknown + return null; + } + time = timeVector[index]; + value = DimensionHandlerUtils.convertObjectToDouble(object); + } + return new SerializablePairLongDouble(time, value); + } + + @Nullable + public static SerializablePairLongDouble readDoublePairFromSelectors( + final BaseLongColumnValueSelector timeSelector, + final BaseObjectColumnValueSelector objectSelector + ) + { + final long time; + final Double value; + + final Object object = objectSelector.getObject(); + if (object instanceof SerializablePairLongDouble) { + // We got a folded object, ignore timeSelector completely, the object has all the info it requires + final SerializablePairLongDouble pair = (SerializablePairLongDouble) object; + time = pair.lhs; + value = pair.rhs; + } else { + // We don't have a folded up object, use timeSelector to get time, and objectSelector to get value + if (timeSelector.isNull()) { + // Don't aggregate null timed value + return null; + } + time = timeSelector.getLong(); + value = DimensionHandlerUtils.convertObjectToDouble(objectSelector.getObject()); + } + return new SerializablePairLongDouble(time, value); + } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java index d40ba750476b..3e38ccfbd65c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.last; import org.apache.druid.query.aggregation.SerializablePairLongDouble; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -29,7 +30,7 @@ /** * Vectorized version of on heap 'last' aggregator for column selectors with type DOUBLE.. */ -public class DoubleLastVectorAggregator extends NumericLastVectorAggregator +public class DoubleLastVectorAggregator extends NumericLastVectorAggregator { double lastValue; @@ -46,25 +47,50 @@ public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorValueS } @Override - void putValue(ByteBuffer buf, int position, int index) + void initValue(ByteBuffer buf, int position) { - lastValue = valueSelector != null ? valueSelector.getDoubleVector()[index] : ((SerializablePairLongDouble) objectSelector.getObjectVector()[index]).getRhs(); - buf.putDouble(position, lastValue); + buf.putDouble(position, 0.0d); } + @Override + void putValue(ByteBuffer buf, int position, Double value) + { + buf.putDouble(position, value); + } + + @Override + void putValue(ByteBuffer buf, int position, VectorValueSelector valueSelector, int index) + { + buf.putDouble(position, valueSelector.getDoubleVector()[index]); + } + + @Override + void putDefaultValue(ByteBuffer buf, int position) + { + buf.putDouble(position, 0.0d); + } + + @Override + Double getValue(ByteBuffer buf, int position) + { + return buf.getDouble(position); + } @Override - public void initValue(ByteBuffer buf, int position) + SerializablePairLongDouble readPairFromVectorSelectors( + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, + int index + ) { - buf.putDouble(position, 0); + return FirstLastUtils.readDoublePairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); } - @Nullable @Override - public Object get(ByteBuffer buf, int position) + SerializablePairLongDouble createPair(long time, @Nullable Double value) { - final boolean rhsNull = isValueNull(buf, position); - return new SerializablePairLongDouble(buf.getLong(position), rhsNull ? null : buf.getDouble(position + VALUE_OFFSET)); + return new SerializablePairLongDouble(time, value); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java index a5500c19cbe8..b8fe6001af57 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java @@ -29,43 +29,70 @@ /** * Vectorized version of on heap 'last' aggregator for column selectors with type FLOAT.. */ -public class FloatLastVectorAggregator extends NumericLastVectorAggregator +public class FloatLastVectorAggregator extends NumericLastVectorAggregator { - float lastValue; public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { super(timeSelector, null, objectSelector); - lastValue = 0; } public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) { super(timeSelector, valueSelector, null); - lastValue = 0; } @Override - void putValue(ByteBuffer buf, int position, int index) + void initValue(ByteBuffer buf, int position) { - lastValue = valueSelector != null ? valueSelector.getFloatVector()[index] : ((SerializablePairLongFloat) objectSelector.getObjectVector()[index]).getRhs(); - buf.putFloat(position, lastValue); + buf.putFloat(position, 0.0f); } + @Override + void putValue(ByteBuffer buf, int position, Float value) + { + buf.putFloat(position, value); + } @Override - public void initValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, VectorValueSelector valueSelector, int index) { - buf.putFloat(position, 0); + buf.putFloat(position, valueSelector.getFloatVector()[index]); } + @Override + void putDefaultValue(ByteBuffer buf, int position) + { + buf.putFloat(position, 0.0f); + } - @Nullable @Override - public Object get(ByteBuffer buf, int position) + Float getValue(ByteBuffer buf, int position) { - final boolean rhsNull = isValueNull(buf, position); - return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); + return buf.getFloat(position); } + + // @Override +// void putValue(ByteBuffer buf, int position, int index) +// { +// lastValue = valueSelector != null ? valueSelector.getFloatVector()[index] : ((SerializablePairLongFloat) objectSelector.getObjectVector()[index]).getRhs(); +// buf.putFloat(position, lastValue); +// } + + +// @Override +// public void initValue(ByteBuffer buf, int position) +// { +// buf.putFloat(position, 0); +// } + + +// @Nullable +// @Override +// public Object get(ByteBuffer buf, int position) +// { +// final boolean rhsNull = isValueNull(buf, position); +// return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); +// } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java index a556c3fea186..733a2535179e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.last; +import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.VectorAggregator; @@ -31,19 +32,34 @@ /** * Base type for vectorized version of on heap 'last' aggregator for primitive numeric column selectors.. */ -public abstract class NumericLastVectorAggregator implements VectorAggregator +public abstract class NumericLastVectorAggregator> + implements VectorAggregator { - static final int NULL_OFFSET = Long.BYTES; - static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; - final VectorValueSelector valueSelector; - final VectorObjectSelector objectSelector; + private static final byte IS_NULL_BYTE = (byte) 1; + private static final byte IS_NOT_NULL_BYTE = (byte) 0; + + static final int NULLITY_OFFSET = Long.BYTES; + static final int VALUE_OFFSET = NULLITY_OFFSET + Byte.BYTES; + + @Nullable + private final VectorValueSelector valueSelector; + @Nullable + private final VectorObjectSelector objectSelector; private final boolean useDefault = NullHandling.replaceWithDefault(); private final VectorValueSelector timeSelector; private long lastTime; - NumericLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector, VectorObjectSelector objectSelector) + NumericLastVectorAggregator( + VectorValueSelector timeSelector, + @Nullable VectorValueSelector valueSelector, + @Nullable VectorObjectSelector objectSelector + ) { + Preconditions.checkArgument( + (valueSelector != null && objectSelector == null) || (valueSelector == null && objectSelector != null), + "exactly one of 'valueSelector' and 'objectSelector' must be provided" + ); this.timeSelector = timeSelector; this.valueSelector = valueSelector; this.objectSelector = objectSelector; @@ -54,82 +70,71 @@ public abstract class NumericLastVectorAggregator implements VectorAggregator public void init(ByteBuffer buf, int position) { buf.putLong(position, Long.MIN_VALUE); - buf.put(position + NULL_OFFSET, useDefault ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE); + buf.put(position + NULLITY_OFFSET, useDefault ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); initValue(buf, position + VALUE_OFFSET); } @Override public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) { - if (timeSelector == null) { - return; - } - - final long[] timeVector = timeSelector.getLongVector(); - Object[] objectsWhichMightBeNumeric = null; - boolean[] nullValueVector = null; - boolean nullAbsent = false; + // If objectSelector isn't null, then the objects might be folded up. If that's the case, whatever's represented by + // the timeSelector doesn't hold any relevance. if (objectSelector != null) { - objectsWhichMightBeNumeric = objectSelector.getObjectVector(); - } else if (valueSelector != null) { - nullValueVector = valueSelector.getNullVector(); - } - - lastTime = buf.getLong(position); + final Object[] maybeFoldedObjects = objectSelector.getObjectVector(); + final boolean[] timeNullityVector = timeSelector.getNullVector(); + final long[] timeVector = timeSelector.getLongVector(); + lastTime = buf.getLong(position); - if (nullValueVector == null) { - nullAbsent = true; - } - - //the time vector is already sorted so the last element would be the latest - //traverse the value vector from the back (for latest) - int index = endRow - 1; - if (!useDefault && !nullAbsent) { - for (int i = endRow - 1; i >= startRow; i--) { - if (!nullValueVector[i]) { - index = i; - break; - } - } - } - - if (objectsWhichMightBeNumeric != null) { - final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; - if (inPair != null && inPair.lhs != null && inPair.lhs >= lastTime) { - lastTime = inPair.lhs; - if (useDefault || inPair.rhs != null) { - updateTimeWithValue(buf, position, lastTime, index); - } else { - updateTimeWithNull(buf, position, lastTime); + for (int index = endRow - 1; index >= startRow; --index) { + PairType pair = readPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); + if (pair != null) { + if (pair.lhs >= lastTime) { + if (pair.rhs != null) { + updateTimeAndValue(buf, position, pair.lhs, pair.rhs); + } else if (useDefault) { + updateTimeAndDefaultValue(buf, position, pair.lhs); + } else { + updateTimeAndNull(buf, position, pair.lhs); + } + } } } } else { - final long latestTime = timeVector[index]; - if (latestTime >= lastTime) { - lastTime = latestTime; - if (useDefault || nullValueVector == null || !nullValueVector[index]) { - updateTimeWithValue(buf, position, lastTime, index); - } else { - updateTimeWithNull(buf, position, lastTime); + // No object selector, no folding present. Check the timeSelector before checking the valueSelector + final boolean[] timeNullityVector = timeSelector.getNullVector(); + final long[] timeVector = timeSelector.getLongVector(); + final boolean[] valueNullityVector = valueSelector.getNullVector(); + int selectedIndex = endRow - 1; + long lastTime = buf.getLong(position); + for (int index = endRow - 1; index >= startRow; --index) { + if (timeNullityVector != null && timeNullityVector[index]) { + // Don't aggregate values where time isn't present + continue; + } + // Find the latest time inside the vector objects + if (timeVector[selectedIndex] >= timeVector[index]) { + selectedIndex = index; + } + + // Compare the selectedIndex's value to the value on the buffer. This way, we write to the buffer only once + // Weeds out empty vectors, where endRow == startRow + if (selectedIndex >= startRow) { + if (timeVector[selectedIndex] >= lastTime) { + // Write the value here + if (valueNullityVector != null && !valueNullityVector[selectedIndex]) { + updateTimeAndValue(buf, position, timeVector[selectedIndex], valueSelector, selectedIndex); + } else if (useDefault) { + updateTimeAndDefaultValue(buf, position, timeVector[selectedIndex]); + } else { + updateTimeAndNull(buf, position, timeVector[index]); + } + } } } } } - /** - * - * Checks if the aggregated value at a position in the buffer is null or not - * - * @param buf byte buffer storing the byte array representation of the aggregate - * @param position offset within the byte buffer at which the current aggregate value is stored - * @return - */ - boolean isValueNull(ByteBuffer buf, int position) - { - return buf.get(position + NULL_OFFSET) == NullHandling.IS_NULL_BYTE; - } - @Override public void aggregate( ByteBuffer buf, @@ -139,86 +144,146 @@ public void aggregate( int positionOffset ) { - if (timeSelector == null) { - return; - } - - final long[] timeVector = timeSelector.getLongVector(); - Object[] objectsWhichMightBeNumeric = null; - boolean[] nulls = null; + // If objectSelector isn't null, then the objects might be folded up. If that's the case, whatever's represented by + // the timeSelector doesn't hold any relevance. We should check for folded objects BEFORE even thinking about looking + // at the timeSelector if (objectSelector != null) { - objectsWhichMightBeNumeric = objectSelector.getObjectVector(); - } else if (valueSelector != null) { - nulls = useDefault ? null : valueSelector.getNullVector(); - } - - - - for (int i = 0; i < numRows; i++) { - int position = positions[i] + positionOffset; - int row = rows == null ? i : rows[i]; - long lastTime = buf.getLong(position); - - if (objectsWhichMightBeNumeric != null) { - final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; - if (useDefault || inPair != null) { - if (inPair.lhs != null && inPair.lhs >= lastTime) { - if (inPair.rhs != null) { - updateTimeWithValue(buf, position, inPair.lhs, row); + final Object[] maybeFoldedObjects = objectSelector.getObjectVector(); + final boolean[] timeNullityVector = timeSelector.getNullVector(); + final long[] timeVector = timeSelector.getLongVector(); + for (int i = 0; i < numRows; ++i) { + int position = positions[i] + positionOffset; + int row = rows == null ? i : rows[i]; + long lastTime = buf.getLong(position); + // All the information about the object would be in the single selector. This method will check the folding of the object selector, + // casting, and everything........ + PairType pair = readPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, row); + if (pair != null) { + if (pair.lhs >= lastTime) { + if (pair.rhs != null) { + updateTimeAndValue(buf, position, pair.lhs, pair.rhs); + } else if (useDefault) { + updateTimeAndDefaultValue(buf, position, pair.lhs); } else { - updateTimeWithNull(buf, position, inPair.lhs); + updateTimeAndNull(buf, position, pair.lhs); } } } - } else { + } + } else { + // No object selector, no folding present. Check the timeSelector before checking the valueSelector + final boolean[] timeNullityVector = timeSelector.getNullVector(); + final long[] timeVector = timeSelector.getLongVector(); + final boolean[] valueNullityVector = valueSelector.getNullVector(); + + for (int i = 0; i < numRows; ++i) { + int position = positions[i] + positionOffset; + int row = rows == null ? i : rows[i]; + long lastTime = buf.getLong(position); + if (timeNullityVector != null && timeNullityVector[row]) { + // Don't aggregate values where time isn't present + continue; + } if (timeVector[row] >= lastTime) { - if (useDefault || nulls == null || !nulls[row]) { - updateTimeWithValue(buf, position, timeVector[row], row); + if (valueNullityVector != null && !valueNullityVector[row]) { + updateTimeAndValue(buf, position, timeVector[row], valueSelector, row); + } else if (useDefault) { + updateTimeAndDefaultValue(buf, position, timeVector[row]); } else { - updateTimeWithNull(buf, position, timeVector[row]); + updateTimeAndNull(buf, position, timeVector[row]); } } } } } - /** - * - * @param buf byte buffer storing the byte array representation of the aggregate - * @param position offset within the byte buffer at which the current aggregate value is stored - * @param time the time to be updated in the buffer as the last time - * @param index he index of the vectorized vector which is the last value - */ - void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + return createPair(buf.getLong(position), null); + } + return createPair(buf.getLong(position), getValue(buf, position + VALUE_OFFSET)); + } + + private void updateTimeAndValue(ByteBuffer buf, int position, long time, RhsType value) { buf.putLong(position, time); - buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, index); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + putValue(buf, position + VALUE_OFFSET, value); } - /** - * - * @param buf byte buffer storing the byte array representation of the aggregate - * @param position offset within the byte buffer at which the current aggregate value is stored - * @param time the time to be updated in the buffer as the last time - */ - void updateTimeWithNull(ByteBuffer buf, int position, long time) + private void updateTimeAndValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + putValue(buf, position + VALUE_OFFSET, valueSelector, index); + } + + private void updateTimeAndDefaultValue(ByteBuffer buf, int position, long time) { buf.putLong(position, time); - buf.put(position + NULL_OFFSET, NullHandling.IS_NULL_BYTE); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + putDefaultValue(buf, position + VALUE_OFFSET); + } + + private void updateTimeAndNull(ByteBuffer buf, int position, long time) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, IS_NULL_BYTE); } /** - *Abstract function which needs to be overridden by subclasses to set the initial value + * Sets the initial value in the byte buffer at the given position + * + * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of + * the serialized pair) */ abstract void initValue(ByteBuffer buf, int position); /** - *Abstract function which needs to be overridden by subclasses to set the - * latest value in the buffer depending on the datatype + * Sets the value at the position. Subclasses can assume that the value isn't null + * + * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of + * the serialized pair) */ - abstract void putValue(ByteBuffer buf, int position, int index); + abstract void putValue(ByteBuffer buf, int position, RhsType value); + + /** + * Sets the value represented by the valueSelector at the given index. A slightly redundant method to {@link #putValue(ByteBuffer, int, Object)} + * to avoid autoboxing for the numeric types. Subclasses can assume that valueSelector.getNullVector[index] is false (i.e. + * the value represented at the index isn't null) + */ + abstract void putValue(ByteBuffer buf, int position, VectorValueSelector valueSelector, int index); + + /** + * Sets the default value for the type in the byte buffer at the given position. It is only used when replaceNullWithDefault = true, + * therefore the callers don't need to handle any other case. It is separate from {@link #initValue} in case the initial value is + * different from the default value. + * + * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of + * the serialized pair) + */ + abstract void putDefaultValue(ByteBuffer buf, int position); + + /** + * Gets the value at the given position. It is okay to box, because we'd be wrapping it in a SerializablePair + * + * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of + * the serialized pair) + */ + abstract RhsType getValue(ByteBuffer buf, int position); + + abstract PairType readPairFromVectorSelectors( + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, + int index + ); + + abstract PairType createPair(long time, @Nullable RhsType value); @Override public void close() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/SingleStringLastDimensionVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/SingleStringLastDimensionVectorAggregator.java index 6b39088faa2e..795236c34368 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/SingleStringLastDimensionVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/SingleStringLastDimensionVectorAggregator.java @@ -54,7 +54,7 @@ public void init(ByteBuffer buf, int position) { buf.putLong(position, Long.MIN_VALUE); buf.put( - position + NumericLastVectorAggregator.NULL_OFFSET, + position + NumericLastVectorAggregator.NULLITY_OFFSET, useDefault ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE ); buf.putInt(position + NumericLastVectorAggregator.VALUE_OFFSET, 0); @@ -78,7 +78,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) if (latestTime > lastTime) { lastTime = latestTime; buf.putLong(position, lastTime); - buf.put(position + NumericLastVectorAggregator.NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.put(position + NumericLastVectorAggregator.NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); buf.putInt(position + NumericLastVectorAggregator.VALUE_OFFSET, valueVector[index]); } } @@ -100,7 +100,7 @@ public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable in if (timeVector[row] > lastTime) { lastTime = timeVector[row]; buf.putLong(position, lastTime); - buf.put(position + NumericLastVectorAggregator.NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.put(position + NumericLastVectorAggregator.NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); buf.putInt(position + NumericLastVectorAggregator.VALUE_OFFSET, values[row]); } } From 55692bf0f67c0b73ac80f7fe43186a914b09f937 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 3 Apr 2024 23:04:21 +0530 Subject: [PATCH 02/10] more stuff --- .../aggregation/first/FirstLastUtils.java | 71 ++++-- .../first/StringFirstLastUtils.java | 20 +- .../first/StringFirstVectorAggregator.java | 17 +- .../last/DoubleLastVectorAggregator.java | 55 +++-- .../last/FloatLastVectorAggregator.java | 73 +++--- .../last/LongLastAggregatorFactory.java | 4 +- .../last/LongLastVectorAggregator.java | 69 ++++-- .../last/NumericLastVectorAggregator.java | 216 ++++++++++-------- .../last/StringLastVectorAggregator.java | 190 +++------------ .../last/DoubleLastVectorAggregatorTest.java | 2 +- .../last/FloatLastVectorAggregatorTest.java | 2 +- .../last/LongLastVectorAggregatorTest.java | 8 +- .../last/StringLastVectorAggregatorTest.java | 4 +- 13 files changed, 375 insertions(+), 356 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java index 18f9e78c91ab..f3225a579925 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java @@ -20,14 +20,13 @@ package org.apache.druid.query.aggregation.first; import org.apache.druid.query.aggregation.SerializablePairLongDouble; -import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.NilColumnValueSelector; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.vector.VectorObjectSelector; -import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -106,29 +105,67 @@ public static SerializablePairLongDouble readDoublePairFromVectorSelectors( } @Nullable - public static SerializablePairLongDouble readDoublePairFromSelectors( - final BaseLongColumnValueSelector timeSelector, - final BaseObjectColumnValueSelector objectSelector + public static SerializablePairLongFloat readFloatPairFromVectorSelectors( + @Nullable boolean[] timeNullityVector, + long[] timeVector, + Object[] objectVector, + int index ) { final long time; - final Double value; + final Float value; - final Object object = objectSelector.getObject(); - if (object instanceof SerializablePairLongDouble) { + final Object object = objectVector[index]; + + if (object instanceof SerializablePairLongFloat) { // We got a folded object, ignore timeSelector completely, the object has all the info it requires - final SerializablePairLongDouble pair = (SerializablePairLongDouble) object; + final SerializablePairLongFloat pair = (SerializablePairLongFloat) object; + // if time == null, don't aggregate + if (pair.lhs == null) { + return null; + } time = pair.lhs; value = pair.rhs; - } else { - // We don't have a folded up object, use timeSelector to get time, and objectSelector to get value - if (timeSelector.isNull()) { - // Don't aggregate null timed value + } else { + if (timeNullityVector != null && timeNullityVector[index]) { + // Donot aggregate pairs where time is unknown return null; } - time = timeSelector.getLong(); - value = DimensionHandlerUtils.convertObjectToDouble(objectSelector.getObject()); + time = timeVector[index]; + value = DimensionHandlerUtils.convertObjectToFloat(object); } - return new SerializablePairLongDouble(time, value); + return new SerializablePairLongFloat(time, value); + } + @Nullable + public static SerializablePairLongLong readLongPairFromVectorSelectors( + @Nullable boolean[] timeNullityVector, + long[] timeVector, + Object[] objectVector, + int index + ) + { + final long time; + final Long value; + + final Object object = objectVector[index]; + + if (object instanceof SerializablePairLongLong) { + // We got a folded object, ignore timeSelector completely, the object has all the info it requires + final SerializablePairLongLong pair = (SerializablePairLongLong) object; + // if time == null, don't aggregate + if (pair.lhs == null) { + return null; + } + time = pair.lhs; + value = pair.rhs; + } else { + if (timeNullityVector != null && timeNullityVector[index]) { + // Donot aggregate pairs where time is unknown + return null; + } + time = timeVector[index]; + value = DimensionHandlerUtils.convertObjectToLong(object); + } + return new SerializablePairLongLong(time, value); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java index ff1113c91132..db0e812aca8e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java @@ -38,25 +38,29 @@ public class StringFirstLastUtils * Return the object at a particular index from the vector selectors. * index of bounds issues is the responsibility of the caller */ + @Nullable public static SerializablePairLongString readPairFromVectorSelectorsAtIndex( - VectorValueSelector timeSelector, - VectorObjectSelector valueSelector, + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, int index ) { final long time; final String string; - final Object object = valueSelector.getObjectVector()[index]; + final Object object = maybeFoldedObjects[index]; + if (object instanceof SerializablePairLongString) { final SerializablePairLongString pair = (SerializablePairLongString) object; time = pair.lhs; string = pair.rhs; - } else if (object != null) { - time = timeSelector.getLongVector()[index]; - string = DimensionHandlerUtils.convertObjectToString(object); } else { - // Don't aggregate nulls. - return null; + if (timeNullityVector != null && timeNullityVector[index]) { + // Donot aggregate pairs where time is unknown + return null; + } + time = timeVector[index]; + string = DimensionHandlerUtils.convertObjectToString(object); } return new SerializablePairLongString(time, string); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java index fd2260b8d665..e4704e944ca3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java @@ -79,9 +79,13 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index], SerializablePairLongString.class); if (foldNeeded) { final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( - timeSelector, - valueSelector, - index + null, + null, + null, + 0 +// timeSelector, +// valueSelector, +// index ); if (inPair != null) { firstTime = buf.getLong(position); @@ -140,9 +144,10 @@ public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable in if (timeVector[row] < firstTime) { if (foldNeeded) { final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( - timeSelector, - valueSelector, - row + null, null, null, 0 +// timeSelector, +// valueSelector, +// row ); if (inPair != null) { if (inPair.lhs < firstTime) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java index 3e38ccfbd65c..18537d668762 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; @@ -28,52 +29,70 @@ import java.nio.ByteBuffer; /** - * Vectorized version of on heap 'last' aggregator for column selectors with type DOUBLE.. + * Vectorized version of on heap 'last' aggregator for column selectors with type DOUBLE. */ public class DoubleLastVectorAggregator extends NumericLastVectorAggregator { - double lastValue; - public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { super(timeSelector, null, objectSelector); - lastValue = 0; } public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) { super(timeSelector, valueSelector, null); - lastValue = 0; } @Override - void initValue(ByteBuffer buf, int position) + public void init(ByteBuffer buf, int position) + { + buf.putLong(position, Long.MIN_VALUE); + buf.put(position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); + buf.putDouble(position + VALUE_OFFSET, 0.0D); + } + + @Nullable + @Override + public Object get(ByteBuffer buf, int position) { - buf.putDouble(position, 0.0d); + long time = buf.getLong(position); + if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + return new SerializablePairLongDouble(time, null); + } + return new SerializablePairLongDouble(time, buf.getDouble(position + VALUE_OFFSET)); } @Override - void putValue(ByteBuffer buf, int position, Double value) + void putValue(ByteBuffer buf, int position, long time, Double value) { - buf.putDouble(position, value); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putDouble(position + VALUE_OFFSET, value); } @Override - void putValue(ByteBuffer buf, int position, VectorValueSelector valueSelector, int index) + void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) { - buf.putDouble(position, valueSelector.getDoubleVector()[index]); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putDouble(position + VALUE_OFFSET, valueSelector.getDoubleVector()[index]); } @Override - void putDefaultValue(ByteBuffer buf, int position) + void putNull(ByteBuffer buf, int position, long time) { - buf.putDouble(position, 0.0d); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); + buf.putDouble(position + VALUE_OFFSET, 0.0D); + } @Override - Double getValue(ByteBuffer buf, int position) + void putDefaultValue(ByteBuffer buf, int position, long time) { - return buf.getDouble(position); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putDouble(position, 0.0D); } @Override @@ -86,11 +105,5 @@ SerializablePairLongDouble readPairFromVectorSelectors( { return FirstLastUtils.readDoublePairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); } - - @Override - SerializablePairLongDouble createPair(long time, @Nullable Double value) - { - return new SerializablePairLongDouble(time, value); - } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java index b8fe6001af57..e327db4d5cc4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java @@ -19,7 +19,9 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.SerializablePairLongFloat; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -43,56 +45,65 @@ public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSe } @Override - void initValue(ByteBuffer buf, int position) + public void init(ByteBuffer buf, int position) { - buf.putFloat(position, 0.0f); + buf.putLong(position, Long.MIN_VALUE); + buf.put(position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); + buf.putFloat(position + VALUE_OFFSET, 0.0F); } + @Nullable @Override - void putValue(ByteBuffer buf, int position, Float value) + public Object get(ByteBuffer buf, int position) { - buf.putFloat(position, value); + long time = buf.getLong(position); + if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + return new SerializablePairLongFloat(time, null); + } + return new SerializablePairLongFloat(time, buf.getFloat(position + VALUE_OFFSET)); } @Override - void putValue(ByteBuffer buf, int position, VectorValueSelector valueSelector, int index) + void putValue(ByteBuffer buf, int position, long time, Float value) { - buf.putFloat(position, valueSelector.getFloatVector()[index]); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putFloat(position + VALUE_OFFSET, value); } @Override - void putDefaultValue(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) { - buf.putFloat(position, 0.0f); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putFloat(position + VALUE_OFFSET, valueSelector.getFloatVector()[index]); } @Override - Float getValue(ByteBuffer buf, int position) + void putNull(ByteBuffer buf, int position, long time) { - return buf.getFloat(position); + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); + buf.putFloat(position + VALUE_OFFSET, 0.0F); } - // @Override -// void putValue(ByteBuffer buf, int position, int index) -// { -// lastValue = valueSelector != null ? valueSelector.getFloatVector()[index] : ((SerializablePairLongFloat) objectSelector.getObjectVector()[index]).getRhs(); -// buf.putFloat(position, lastValue); -// } - - -// @Override -// public void initValue(ByteBuffer buf, int position) -// { -// buf.putFloat(position, 0); -// } - + @Override + void putDefaultValue(ByteBuffer buf, int position, long time) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putFloat(position + VALUE_OFFSET, 0.0F); + } -// @Nullable -// @Override -// public Object get(ByteBuffer buf, int position) -// { -// final boolean rhsNull = isValueNull(buf, position); -// return new SerializablePairLongFloat(buf.getLong(position), rhsNull ? null : buf.getFloat(position + VALUE_OFFSET)); -// } + @Override + SerializablePairLongFloat readPairFromVectorSelectors( + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, + int index + ) + { + return FirstLastUtils.readFloatPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); + } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java index 492df2136492..15951808c763 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -149,9 +149,7 @@ public boolean canVectorize(ColumnInspector columnInspector) } @Override - public VectorAggregator factorizeVector( - VectorColumnSelectorFactory columnSelectorFactory - ) + public VectorAggregator factorizeVector(VectorColumnSelectorFactory columnSelectorFactory) { VectorValueSelector timeSelector = columnSelectorFactory.makeValueSelector(timeColumn); ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(fieldName); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java index 65fadf549a76..70943b1d4419 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java @@ -19,7 +19,9 @@ package org.apache.druid.query.aggregation.last; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.SerializablePairLongLong; +import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -29,44 +31,77 @@ /** * Vectorized version of on heap 'last' aggregator for column selectors with type LONG.. */ -public class LongLastVectorAggregator extends NumericLastVectorAggregator +public class LongLastVectorAggregator extends NumericLastVectorAggregator { - long lastValue; - public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) { super(timeSelector, null, objectSelector); - lastValue = 0; } public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) { super(timeSelector, valueSelector, null); - lastValue = 0; } @Override - public void initValue(ByteBuffer buf, int position) + public void init(ByteBuffer buf, int position) { - buf.putLong(position, 0); + buf.putLong(position, Long.MIN_VALUE); + buf.put(position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); + buf.putLong(position + VALUE_OFFSET, 0L); } + @Nullable @Override - void putValue(ByteBuffer buf, int position, int index) + public Object get(ByteBuffer buf, int position) { - lastValue = valueSelector != null ? valueSelector.getLongVector()[index] : ((SerializablePairLongLong) objectSelector.getObjectVector()[index]).getRhs(); - buf.putLong(position, lastValue); + long time = buf.getLong(position); + if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + return new SerializablePairLongLong(time, null); + } + return new SerializablePairLongLong(time, buf.getLong(position + VALUE_OFFSET)); } + @Override + void putValue(ByteBuffer buf, int position, long time, Long value) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putLong(position + VALUE_OFFSET, value); + } - /** - * @return The primitive object stored at the position in the buffer. - */ - @Nullable @Override - public Object get(ByteBuffer buf, int position) + void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putLong(position + VALUE_OFFSET, valueSelector.getLongVector()[index]); + } + + @Override + void putNull(ByteBuffer buf, int position, long time) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); + buf.putLong(position + VALUE_OFFSET, 0L); + } + + @Override + void putDefaultValue(ByteBuffer buf, int position, long time) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putLong(position + VALUE_OFFSET, 0L); + } + + @Override + SerializablePairLongLong readPairFromVectorSelectors( + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, + int index + ) { - final boolean rhsNull = isValueNull(buf, position); - return new SerializablePairLongLong(buf.getLong(position), rhsNull ? null : buf.getLong(position + VALUE_OFFSET)); + return FirstLastUtils.readLongPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java index 733a2535179e..baf096821c9a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java @@ -35,23 +35,30 @@ public abstract class NumericLastVectorAggregator> implements VectorAggregator { - private static final byte IS_NULL_BYTE = (byte) 1; - private static final byte IS_NOT_NULL_BYTE = (byte) 0; - - static final int NULLITY_OFFSET = Long.BYTES; - static final int VALUE_OFFSET = NULLITY_OFFSET + Byte.BYTES; + protected static final byte IS_NULL_BYTE = (byte) 0; + protected static final byte IS_NOT_NULL_BYTE = (byte) 1; + protected static final int NULLITY_OFFSET = Long.BYTES; + protected static final int VALUE_OFFSET = NULLITY_OFFSET + Byte.BYTES; + @Nullable + private final VectorValueSelector timeSelector; @Nullable private final VectorValueSelector valueSelector; @Nullable private final VectorObjectSelector objectSelector; private final boolean useDefault = NullHandling.replaceWithDefault(); - private final VectorValueSelector timeSelector; - private long lastTime; + /** + * TODO(laksh): valueSelector isn't used much here, only checked for nullity. However while calling the methods of the subclasses, + * it gets used because it is clearer to know which selector is getting used. This gets used + * + * timeSelector can be null, however aggregate functions are no-op then. Weird, since all numeric versions supply the selector, + * only the case when the aggregator's capabilities are not present in the string version do we hit this case (not sure why this is + * a possibility, and what benefit does it provides) + */ NumericLastVectorAggregator( - VectorValueSelector timeSelector, + @Nullable VectorValueSelector timeSelector, @Nullable VectorValueSelector valueSelector, @Nullable VectorObjectSelector objectSelector ) @@ -63,50 +70,64 @@ public abstract class NumericLastVectorAggregator= startRow; --index) { PairType pair = readPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); if (pair != null) { - if (pair.lhs >= lastTime) { - if (pair.rhs != null) { - updateTimeAndValue(buf, position, pair.lhs, pair.rhs); - } else if (useDefault) { - updateTimeAndDefaultValue(buf, position, pair.lhs); - } else { - updateTimeAndNull(buf, position, pair.lhs); - } + if (selectedPair == null) { + selectedPair = pair; + } else if (pair.lhs >= selectedPair.lhs) { + selectedPair = pair; + } + } + } + // Something's been selected of the row vector + if (selectedPair != null) { + // Compare the latest value of the folded up row vector to the latest value in the buffer + if (selectedPair.lhs >= buf.getLong(position)) { + if (selectedPair.rhs != null) { + putValue(buf, position, selectedPair.lhs, selectedPair.rhs); + } else if (useDefault) { + putDefaultValue(buf, position, selectedPair.lhs); + } else { + putNull(buf, position, selectedPair.lhs); } } } + } else { // No object selector, no folding present. Check the timeSelector before checking the valueSelector final boolean[] timeNullityVector = timeSelector.getNullVector(); final long[] timeVector = timeSelector.getLongVector(); final boolean[] valueNullityVector = valueSelector.getNullVector(); int selectedIndex = endRow - 1; - long lastTime = buf.getLong(position); for (int index = endRow - 1; index >= startRow; --index) { if (timeNullityVector != null && timeNullityVector[index]) { // Don't aggregate values where time isn't present @@ -120,14 +141,14 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) // Compare the selectedIndex's value to the value on the buffer. This way, we write to the buffer only once // Weeds out empty vectors, where endRow == startRow if (selectedIndex >= startRow) { - if (timeVector[selectedIndex] >= lastTime) { + if (timeVector[selectedIndex] >= buf.getLong(position)) { // Write the value here if (valueNullityVector != null && !valueNullityVector[selectedIndex]) { - updateTimeAndValue(buf, position, timeVector[selectedIndex], valueSelector, selectedIndex); + putValue(buf, position, timeVector[selectedIndex], valueSelector, selectedIndex); } else if (useDefault) { - updateTimeAndDefaultValue(buf, position, timeVector[selectedIndex]); + putDefaultValue(buf, position, timeVector[selectedIndex]); } else { - updateTimeAndNull(buf, position, timeVector[index]); + putNull(buf, position, timeVector[index]); } } } @@ -144,6 +165,12 @@ public void aggregate( int positionOffset ) { + // Not a normal case, and this doesn't affect the folding. timeSelectors should be present (albeit irrelevent) when folding. + // timeSelector == null means that the aggregating column's capabilities aren't known, and it only happens for a special case + // while building string aggregator + if (timeSelector == null) { + return; + } // If objectSelector isn't null, then the objects might be folded up. If that's the case, whatever's represented by // the timeSelector doesn't hold any relevance. We should check for folded objects BEFORE even thinking about looking @@ -162,11 +189,11 @@ public void aggregate( if (pair != null) { if (pair.lhs >= lastTime) { if (pair.rhs != null) { - updateTimeAndValue(buf, position, pair.lhs, pair.rhs); + putValue(buf, position, pair.lhs, pair.rhs); } else if (useDefault) { - updateTimeAndDefaultValue(buf, position, pair.lhs); + putDefaultValue(buf, position, pair.lhs); } else { - updateTimeAndNull(buf, position, pair.lhs); + putNull(buf, position, pair.lhs); } } } @@ -187,94 +214,91 @@ public void aggregate( } if (timeVector[row] >= lastTime) { if (valueNullityVector != null && !valueNullityVector[row]) { - updateTimeAndValue(buf, position, timeVector[row], valueSelector, row); + putValue(buf, position, timeVector[row], valueSelector, row); } else if (useDefault) { - updateTimeAndDefaultValue(buf, position, timeVector[row]); + putDefaultValue(buf, position, timeVector[row]); } else { - updateTimeAndNull(buf, position, timeVector[row]); + putNull(buf, position, timeVector[row]); } } } } } +// +// @Nullable +// @Override +// public Object get(ByteBuffer buf, int position) +// { +// if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { +// return createPair(buf.getLong(position), null); +// } +// return createPair(buf.getLong(position), getValue(buf, position + VALUE_OFFSET)); +// } - @Nullable - @Override - public Object get(ByteBuffer buf, int position) - { - if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { - return createPair(buf.getLong(position), null); - } - return createPair(buf.getLong(position), getValue(buf, position + VALUE_OFFSET)); - } - - private void updateTimeAndValue(ByteBuffer buf, int position, long time, RhsType value) - { - buf.putLong(position, time); - buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, value); - } - - private void updateTimeAndValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) - { - buf.putLong(position, time); - buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, valueSelector, index); - } - - private void updateTimeAndDefaultValue(ByteBuffer buf, int position, long time) - { - buf.putLong(position, time); - buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putDefaultValue(buf, position + VALUE_OFFSET); - } - - private void updateTimeAndNull(ByteBuffer buf, int position, long time) - { - buf.putLong(position, time); - buf.put(position + NULLITY_OFFSET, IS_NULL_BYTE); - } - - /** - * Sets the initial value in the byte buffer at the given position - * - * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of - * the serialized pair) - */ - abstract void initValue(ByteBuffer buf, int position); +// private void updateTimeAndValue(ByteBuffer buf, int position, long time, RhsType value) +// { +// buf.putLong(position, time); +// buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); +// putValue(buf, position + VALUE_OFFSET, value); +// } +// +// private void updateTimeAndValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) +// { +// buf.putLong(position, time); +// buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); +// putValue(buf, position + VALUE_OFFSET, valueSelector, index); +// } +// +// private void updateTimeAndDefaultValue(ByteBuffer buf, int position, long time) +// { +// buf.putLong(position, time); +// buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); +// putDefaultValue(buf, position + VALUE_OFFSET); +// } +// +// private void updateTimeAndNull(ByteBuffer buf, int position, long time) +// { +// buf.putLong(position, time); +// buf.put(position + NULLITY_OFFSET, IS_NULL_BYTE); +// } /** * Sets the value at the position. Subclasses can assume that the value isn't null * * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of * the serialized pair) + * + * It is only used if objectSelector is supplied */ - abstract void putValue(ByteBuffer buf, int position, RhsType value); + abstract void putValue(ByteBuffer buf, int position, long time, RhsType value); /** - * Sets the value represented by the valueSelector at the given index. A slightly redundant method to {@link #putValue(ByteBuffer, int, Object)} + * Sets the value represented by the valueSelector at the given index. A slightly redundant method to {@link #putValue(ByteBuffer, int, long, Object)} * to avoid autoboxing for the numeric types. Subclasses can assume that valueSelector.getNullVector[index] is false (i.e. * the value represented at the index isn't null) + * + * It is used if valueSelector is supplied */ - abstract void putValue(ByteBuffer buf, int position, VectorValueSelector valueSelector, int index); + abstract void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index); /** * Sets the default value for the type in the byte buffer at the given position. It is only used when replaceNullWithDefault = true, - * therefore the callers don't need to handle any other case. It is separate from {@link #initValue} in case the initial value is - * different from the default value. + * therefore the callers don't need to handle any other case. * * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of * the serialized pair) */ - abstract void putDefaultValue(ByteBuffer buf, int position); + abstract void putDefaultValue(ByteBuffer buf, int position, long time); - /** - * Gets the value at the given position. It is okay to box, because we'd be wrapping it in a SerializablePair - * - * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of - * the serialized pair) - */ - abstract RhsType getValue(ByteBuffer buf, int position); + abstract void putNull(ByteBuffer buf, int position, long time); + +// /** +// * Gets the value at the given position. It is okay to box, because we'd be wrapping it in a SerializablePair +// * +// * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of +// * the serialized pair) +// */ +// abstract RhsType getValue(ByteBuffer buf, int position); abstract PairType readPairFromVectorSelectors( boolean[] timeNullityVector, @@ -283,7 +307,7 @@ abstract PairType readPairFromVectorSelectors( int index ); - abstract PairType createPair(long time, @Nullable RhsType value); +// abstract PairType createPair(long time, @Nullable RhsType value); @Override public void close() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java index 09ef10572b59..0bd4a6e2d491 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java @@ -1,55 +1,33 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.druid.query.aggregation.last; -import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.error.DruidException; import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.query.aggregation.first.StringFirstLastUtils; -import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; -public class StringLastVectorAggregator implements VectorAggregator +public class StringLastVectorAggregator extends NumericLastVectorAggregator { + // Initialized with MIN_VALUE instead of DateTimes.MIN.getMillis(), as it can be a custom timeSelector (provided via LATEST_BY) + // that has a lower min than the times.min private static final SerializablePairLongString INIT = new SerializablePairLongString( - DateTimes.MIN.getMillis(), + Long.MIN_VALUE, null ); - private final VectorValueSelector timeSelector; - private final VectorObjectSelector valueSelector; + private final int maxStringBytes; - protected long lastTime; public StringLastVectorAggregator( @Nullable final VectorValueSelector timeSelector, - final VectorObjectSelector valueSelector, + final VectorObjectSelector objectSelector, final int maxStringBytes ) { - this.timeSelector = timeSelector; - this.valueSelector = valueSelector; + super(timeSelector, null, objectSelector); this.maxStringBytes = maxStringBytes; } @@ -59,142 +37,50 @@ public void init(ByteBuffer buf, int position) StringFirstLastUtils.writePair(buf, position, INIT, maxStringBytes); } + @Nullable @Override - public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) + public Object get(ByteBuffer buf, int position) { - if (timeSelector == null) { - return; - } - final long[] times = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final Object[] objectsWhichMightBeStrings = valueSelector.getObjectVector(); - - lastTime = buf.getLong(position); - int index; - for (int i = endRow - 1; i >= startRow; i--) { - if (objectsWhichMightBeStrings[i] == null) { - continue; - } - if (times[i] <= lastTime) { - continue; - } - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - index = i; - final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index], SerializablePairLongString.class); - if (foldNeeded) { - // Less efficient code path when folding is a possibility (we must read the value selector first just in case - // it's a foldable object). - final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( - timeSelector, - valueSelector, - index - ); - if (inPair != null) { - final long lastTime = buf.getLong(position); - if (inPair.lhs >= lastTime) { - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(inPair.lhs, inPair.rhs), - maxStringBytes - ); - } - } - } else { - final long time = times[index]; - - if (time >= lastTime) { - final String value = DimensionHandlerUtils.convertObjectToString(objectsWhichMightBeStrings[index]); - lastTime = time; - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(time, value), - maxStringBytes - ); - } - } - } - + return StringFirstLastUtils.readPair(buf, position); } @Override - public void aggregate( - ByteBuffer buf, - int numRows, - int[] positions, - @Nullable int[] rows, - int positionOffset - ) + void putValue(ByteBuffer buf, int position, long time, String value) { - if (timeSelector == null) { - return; - } - final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final Object[] objectsWhichMightBeStrings = valueSelector.getObjectVector(); + StringFirstLastUtils.writePair(buf, position, new SerializablePairLongString(time, value), maxStringBytes); + } - // iterate once over the object vector to find first non null element and - // determine if the type is Pair or not - boolean foldNeeded = false; - for (Object obj : objectsWhichMightBeStrings) { - if (obj != null) { - foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, SerializablePairLongString.class); - break; - } - } + @Override + void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + { + throw DruidException.defensive("This variant is not applicable to the StringLastVectorAggregator"); + } - for (int i = 0; i < numRows; i++) { - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - int position = positions[i] + positionOffset; - int row = rows == null ? i : rows[i]; - long lastTime = buf.getLong(position); - if (timeVector[row] >= lastTime) { - if (foldNeeded) { - final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( - timeSelector, - valueSelector, - row - ); - if (inPair != null) { - if (inPair.lhs >= lastTime) { - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(inPair.lhs, inPair.rhs), - maxStringBytes - ); - } - } - } else { - final String value = DimensionHandlerUtils.convertObjectToString(objectsWhichMightBeStrings[row]); - lastTime = timeVector[row]; - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(lastTime, value), - maxStringBytes - ); - } - } - } + @Override + void putDefaultValue(ByteBuffer buf, int position, long time) + { + StringFirstLastUtils.writePair( + buf, + position, + new SerializablePairLongString(time, NullHandling.defaultStringValue()), + maxStringBytes + ); } - @Nullable @Override - public Object get(ByteBuffer buf, int position) + void putNull(ByteBuffer buf, int position, long time) { - return StringFirstLastUtils.readPair(buf, position); + StringFirstLastUtils.writePair(buf, position, new SerializablePairLongString(time, null), maxStringBytes); } @Override - public void close() + SerializablePairLongString readPairFromVectorSelectors( + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, + int index + ) { - // nothing to close + return StringFirstLastUtils.readPairFromVectorSelectorsAtIndex(timeNullityVector, timeVector, maybeFoldedObjects, index); } } - diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java index d3125f190e1c..93ad62b6c94f 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java @@ -233,7 +233,7 @@ public void testFactory() @Test public void initValueShouldInitZero() { - target.initValue(buf, 0); + target.init(buf, 0); double initVal = buf.getDouble(0); Assert.assertEquals(0, initVal, EPSILON); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java index 957d585e5fd9..b522362bdaa1 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java @@ -237,7 +237,7 @@ public void testFactory() @Test public void initValueShouldBeZero() { - target.initValue(buf, 0); + target.init(buf, 0); float initVal = buf.getFloat(0); Assert.assertEquals(0.0f, initVal, EPSILON); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java index 6cc8bfa9f5a2..48b22f6a18b4 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java @@ -44,6 +44,11 @@ import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; +// Check with empty arrays, nulls +// Check with folded up objects, and non-folded up objects +// Check with coercing objects +// Check both the functions that they work properly +// Check with pre-existing value on the buffer, and no pre-existing value on the buffer public class LongLastVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; @@ -235,7 +240,7 @@ public void testFactory() @Test public void initValueShouldInitZero() { - target.initValue(buf, 0); + target.init(buf, 0); long initVal = buf.getLong(0); Assert.assertEquals(0, initVal); } @@ -243,6 +248,7 @@ public void initValueShouldInitZero() @Test public void aggregate() { + target.init(buf, 0); target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); Assert.assertEquals(pairs[3].lhs.longValue(), result.lhs.longValue()); diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java index da79faae3c15..50d7d63a4df8 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java @@ -358,11 +358,11 @@ public void testStringLastOnNonStringColumns() } @Test - public void initValueShouldBeMinDate() + public void initValueShouldBeMinLong() { target.init(buf, 0); long initVal = buf.getLong(0); - Assert.assertEquals(DateTimes.MIN.getMillis(), initVal); + Assert.assertEquals(Long.MIN_VALUE, initVal); } @Test From 1fab930589ef1899d92777de9e225a28dbfd16bf Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 4 Apr 2024 03:02:51 +0530 Subject: [PATCH 03/10] move stuff, remove redundancy --- .../common/task/CompactionTaskTest.java | 4 +- .../druid/jackson/AggregatorsModule.java | 20 +- .../any/StringAnyAggregatorFactory.java | 2 +- .../first/NumericFirstVectorAggregator.java | 225 ------------------ ...eStringFirstDimensionVectorAggregator.java | 125 ---------- .../first/StringFirstVectorAggregator.java | 189 --------------- .../DoubleFirstLastVectorAggregator.java} | 38 +-- .../{first => firstlast}/FirstLastUtils.java | 2 +- .../FirstLastVectorAggregator.java} | 119 +++------ .../FloatFirstLastVectorAggregator.java} | 38 +-- .../LongFirstLastVectorAggregator.java} | 38 +-- .../firstlast/SelectionPredicate.java | 28 +++ ...ingFirstLastDimensionVectorAggregator.java | 189 +++++++++++++++ .../StringFirstLastUtils.java | 4 +- .../StringFirstLastVectorAggregator.java} | 43 +++- .../first/DoubleFirstAggregator.java | 2 +- .../first/DoubleFirstAggregatorFactory.java | 3 +- .../first/DoubleFirstBufferAggregator.java | 2 +- .../first/DoubleFirstVectorAggregator.java | 40 +--- .../first/FloatFirstAggregator.java | 2 +- .../first/FloatFirstAggregatorFactory.java | 3 +- .../first/FloatFirstBufferAggregator.java | 2 +- .../first/FloatFirstVectorAggregator.java | 39 +-- .../first/GenericFirstAggregateCombiner.java | 2 +- .../first/LongFirstAggregator.java | 2 +- .../first/LongFirstAggregatorFactory.java | 3 +- .../first/LongFirstBufferAggregator.java | 2 +- .../first/LongFirstVectorAggregator.java | 38 +-- .../first/NumericFirstAggregator.java | 2 +- .../first/NumericFirstBufferAggregator.java | 2 +- ...eStringFirstDimensionVectorAggregator.java | 38 +++ .../first/StringFirstAggregator.java | 3 +- .../first/StringFirstAggregatorFactory.java | 3 +- .../first/StringFirstBufferAggregator.java | 3 +- .../StringFirstFoldingAggregatorFactory.java | 2 +- .../first/StringFirstVectorAggregator.java | 39 +++ .../last/DoubleLastAggregator.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 6 +- .../last/DoubleLastBufferAggregator.java | 2 +- .../last/DoubleLastVectorAggregator.java | 39 +++ .../last/FloatLastAggregator.java | 2 +- .../last/FloatLastAggregatorFactory.java | 6 +- .../last/FloatLastBufferAggregator.java | 2 +- .../last/FloatLastVectorAggregator.java | 39 +++ .../last/GenericLastAggregateCombiner.java | 2 +- .../last/LongLastAggregator.java | 2 +- .../last/LongLastAggregatorFactory.java | 6 +- .../last/LongLastBufferAggregator.java | 2 +- .../last/LongLastVectorAggregator.java | 38 +++ .../last/NumericLastAggregator.java | 4 +- .../last/NumericLastBufferAggregator.java | 4 +- ...leStringLastDimensionVectorAggregator.java | 38 +++ .../last/StringLastAggregator.java | 4 +- .../last/StringLastAggregatorFactory.java | 13 +- .../last/StringLastBufferAggregator.java | 4 +- .../StringLastFoldingAggregatorFactory.java | 2 +- .../last/StringLastVectorAggregator.java | 39 +++ ...leStringLastDimensionVectorAggregator.java | 124 ---------- .../aggregation/AggregatorFactoryTest.java | 16 +- .../first/DoubleFirstAggregationTest.java | 2 +- .../DoubleFirstVectorAggregationTest.java | 9 +- .../first/FloatFirstAggregationTest.java | 2 +- .../FloatFirstVectorAggregationTest.java | 9 +- .../first/LongFirstAggregationTest.java | 2 +- .../first/LongFirstVectorAggregationTest.java | 9 +- .../first/StringFirstAggregationTest.java | 2 +- .../StringFirstBufferAggregatorTest.java | 2 +- .../first/StringFirstLastUtilsTest.java | 3 +- .../first/StringFirstTimeseriesQueryTest.java | 2 +- .../StringFirstVectorAggregatorTest.java | 2 +- .../last/DoubleLastAggregationTest.java | 2 +- .../last/DoubleLastVectorAggregatorTest.java | 2 +- .../last/FloatLastAggregationTest.java | 2 +- .../last/FloatLastVectorAggregatorTest.java | 7 +- .../last/LongLastAggregationTest.java | 2 +- .../last/LongLastVectorAggregatorTest.java | 8 +- .../last/StringLastAggregationTest.java | 2 +- .../last/StringLastBufferAggregatorTest.java | 2 +- .../last/StringLastTimeseriesQueryTest.java | 2 +- .../last/StringLastVectorAggregatorTest.java | 3 +- ...nalizingFieldAccessPostAggregatorTest.java | 2 +- .../GroupByQueryQueryToolChestTest.java | 8 +- .../query/groupby/GroupByQueryRunnerTest.java | 4 +- .../TimeseriesQueryQueryToolChestTest.java | 2 +- .../timeseries/TimeseriesQueryRunnerTest.java | 4 +- .../topn/TopNQueryQueryToolChestTest.java | 8 +- .../druid/query/topn/TopNQueryRunnerTest.java | 10 +- .../query/topn/UnnestTopNQueryRunnerTest.java | 2 +- .../druid/segment/IndexMergerRollupTest.java | 16 +- .../EarliestLatestAnySqlAggregator.java | 16 +- .../druid/sql/calcite/CalciteQueryTest.java | 16 +- .../sql/calcite/CalciteSimpleQueryTest.java | 4 +- .../sql/calcite/util/TestDataBuilder.java | 10 +- 93 files changed, 811 insertions(+), 1059 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java delete mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java delete mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{last/DoubleLastVectorAggregator.java => firstlast/DoubleFirstLastVectorAggregator.java} (67%) rename processing/src/main/java/org/apache/druid/query/aggregation/{first => firstlast}/FirstLastUtils.java (99%) rename processing/src/main/java/org/apache/druid/query/aggregation/{last/NumericLastVectorAggregator.java => firstlast/FirstLastVectorAggregator.java} (70%) rename processing/src/main/java/org/apache/druid/query/aggregation/{last/FloatLastVectorAggregator.java => firstlast/FloatFirstLastVectorAggregator.java} (67%) rename processing/src/main/java/org/apache/druid/query/aggregation/{last/LongLastVectorAggregator.java => firstlast/LongFirstLastVectorAggregator.java} (67%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{first => firstlast}/StringFirstLastUtils.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{last/StringLastVectorAggregator.java => firstlast/StringFirstLastVectorAggregator.java} (54%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/DoubleFirstAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/DoubleFirstAggregatorFactory.java (98%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/DoubleFirstBufferAggregator.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/DoubleFirstVectorAggregator.java (50%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/FloatFirstAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/FloatFirstAggregatorFactory.java (98%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/FloatFirstBufferAggregator.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/FloatFirstVectorAggregator.java (51%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/GenericFirstAggregateCombiner.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/LongFirstAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/LongFirstAggregatorFactory.java (98%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/LongFirstBufferAggregator.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/LongFirstVectorAggregator.java (51%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/NumericFirstAggregator.java (98%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/NumericFirstBufferAggregator.java (98%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/first/SingleStringFirstDimensionVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstAggregatorFactory.java (98%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstBufferAggregator.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstFoldingAggregatorFactory.java (96%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/DoubleLastAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/DoubleLastAggregatorFactory.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/DoubleLastBufferAggregator.java (97%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/FloatLastAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/FloatLastAggregatorFactory.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/FloatLastBufferAggregator.java (97%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/GenericLastAggregateCombiner.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/LongLastAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/LongLastAggregatorFactory.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/LongLastBufferAggregator.java (97%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/NumericLastAggregator.java (95%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/NumericLastBufferAggregator.java (96%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/last/SingleStringLastDimensionVectorAggregator.java rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastAggregator.java (96%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastAggregatorFactory.java (94%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastBufferAggregator.java (97%) rename processing/src/main/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastFoldingAggregatorFactory.java (96%) create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregator.java delete mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/last/SingleStringLastDimensionVectorAggregator.java rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/DoubleFirstAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/DoubleFirstVectorAggregationTest.java (96%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/FloatFirstAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/FloatFirstVectorAggregationTest.java (97%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/LongFirstAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/LongFirstVectorAggregationTest.java (96%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstBufferAggregatorTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstLastUtilsTest.java (94%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstTimeseriesQueryTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/first/StringFirstVectorAggregatorTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/DoubleLastAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/DoubleLastVectorAggregatorTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/FloatLastAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/FloatLastVectorAggregatorTest.java (97%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/LongLastAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/LongLastVectorAggregatorTest.java (97%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastAggregationTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastBufferAggregatorTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastTimeseriesQueryTest.java (99%) rename processing/src/test/java/org/apache/druid/query/aggregation/{ => firstlast}/last/StringLastVectorAggregatorTest.java (99%) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index c90c08349c4d..6d472f4b1800 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -95,8 +95,8 @@ import org.apache.druid.query.aggregation.FloatMinAggregatorFactory; import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.DoubleDimensionHandler; import org.apache.druid.segment.IndexIO; diff --git a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java index dddf39956edb..f7aca511e17d 100644 --- a/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java +++ b/processing/src/main/java/org/apache/druid/jackson/AggregatorsModule.java @@ -49,20 +49,20 @@ import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.StringAnyAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstFoldingAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstFoldingAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastFoldingAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import org.apache.druid.query.aggregation.hyperloglog.PreComputedHyperUniquesSerde; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastFoldingAggregatorFactory; import org.apache.druid.query.aggregation.mean.DoubleMeanAggregatorFactory; import org.apache.druid.query.aggregation.mean.DoubleMeanHolder; import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java index 67682bfde9dc..b1590381090d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/any/StringAnyAggregatorFactory.java @@ -27,7 +27,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorUtil; import org.apache.druid.query.aggregation.BufferAggregator; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.segment.ColumnInspector; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java deleted file mode 100644 index 6f6bfec014c1..000000000000 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/NumericFirstVectorAggregator.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation.first; - -import org.apache.druid.collections.SerializablePair; -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.segment.vector.VectorObjectSelector; -import org.apache.druid.segment.vector.VectorValueSelector; - -import javax.annotation.Nullable; -import java.nio.ByteBuffer; - -/** - * Class for vectorized version of first/earliest aggregator over numeric types - */ -public abstract class NumericFirstVectorAggregator implements VectorAggregator -{ - static final int NULL_OFFSET = Long.BYTES; - static final int VALUE_OFFSET = NULL_OFFSET + Byte.BYTES; - final VectorObjectSelector objectSelector; - final VectorValueSelector valueSelector; - private final boolean useDefault = NullHandling.replaceWithDefault(); - private final VectorValueSelector timeSelector; - private long firstTime; - - NumericFirstVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector, VectorObjectSelector objectSelector) - { - this.timeSelector = timeSelector; - this.objectSelector = objectSelector; - this.valueSelector = valueSelector; - firstTime = Long.MAX_VALUE; - } - - @Override - public void init(ByteBuffer buf, int position) - { - buf.putLong(position, Long.MAX_VALUE); - buf.put(position + NULL_OFFSET, useDefault ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE); - initValue(buf, position + VALUE_OFFSET); - } - - @Override - public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) - { - final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - - Object[] objectsWhichMightBeNumeric = null; - boolean[] nullValueVector = null; - - if (objectSelector != null) { - objectsWhichMightBeNumeric = objectSelector.getObjectVector(); - } else if (valueSelector != null) { - nullValueVector = valueSelector.getNullVector(); - } - - firstTime = buf.getLong(position); - - // the time vector is already sorted - // if earliest is on the default time dimension - // but if earliest uses earliest_by it might use a secondary timestamp - // which is not sorted. For correctness, we need to go over all elements. - // A possible optimization here is to have 2 paths one for earliest where - // we can take advantage of the sorted nature of time - // and the earliest_by where we have to go over all elements. - - int index; - for (int i = startRow; i < endRow; i++) { - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - - if (timeVector[i] >= firstTime) { - continue; - } - index = i; - - if (objectsWhichMightBeNumeric != null) { - final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[index]; - if (inPair != null && inPair.lhs != null && inPair.lhs < firstTime) { - firstTime = inPair.lhs; - if (useDefault || inPair.rhs != null) { - updateTimeWithValue(buf, position, firstTime, index); - } else { - updateTimeWithNull(buf, position, firstTime); - } - } - } else { - final long earliestTime = timeVector[index]; - - if (earliestTime < firstTime) { - firstTime = earliestTime; - if (useDefault || nullValueVector == null || !nullValueVector[index]) { - updateTimeWithValue(buf, position, earliestTime, index); - } else { - updateTimeWithNull(buf, position, earliestTime); - } - } - } - } - } - - /** - * - * Checks if the aggregated value at a position in the buffer is null or not - * - * @param buf byte buffer storing the byte array representation of the aggregate - * @param position offset within the byte buffer at which the current aggregate value is stored - * @return - */ - boolean isValueNull(ByteBuffer buf, int position) - { - return buf.get(position + NULL_OFFSET) == NullHandling.IS_NULL_BYTE; - } - - @Override - public void aggregate( - ByteBuffer buf, - int numRows, - int[] positions, - @Nullable int[] rows, - int positionOffset - ) - { - final long[] timeVector = timeSelector.getLongVector(); - - Object[] objectsWhichMightBeNumeric = null; - boolean[] nulls = null; - if (objectSelector != null) { - objectsWhichMightBeNumeric = objectSelector.getObjectVector(); - } else if (valueSelector != null) { - nulls = useDefault ? null : valueSelector.getNullVector(); - } - - for (int i = 0; i < numRows; i++) { - int position = positions[i] + positionOffset; - int row = rows == null ? i : rows[i]; - firstTime = buf.getLong(position); - - if (objectsWhichMightBeNumeric != null) { - final SerializablePair inPair = (SerializablePair) objectsWhichMightBeNumeric[row]; - if (useDefault || inPair != null) { - if (inPair.lhs != null && inPair.lhs < firstTime) { - if (inPair.rhs != null) { - updateTimeWithValue(buf, position, inPair.lhs, row); - } else { - updateTimeWithNull(buf, position, inPair.lhs); - } - } - } - } else { - if (timeVector[row] < firstTime) { - if (useDefault || nulls == null || !nulls[row]) { - updateTimeWithValue(buf, position, timeVector[row], row); - } else { - updateTimeWithNull(buf, position, timeVector[row]); - } - } - } - } - } - - /** - * Updates the time and the non null values to the appropriate position in buffer - * - * @param buf byte buffer storing the byte array representation of the aggregate - * @param position offset within the byte buffer at which the current aggregate value is stored - * @param time the time to be updated in the buffer as the first time - * @param index he index of the vectorized vector which is the first value - */ - void updateTimeWithValue(ByteBuffer buf, int position, long time, int index) - { - buf.putLong(position, time); - buf.put(position + NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - putValue(buf, position + VALUE_OFFSET, index); - } - - /** - * Updates the time only to the appropriate position in buffer as the value is null - * - * @param buf byte buffer storing the byte array representation of the aggregate - * @param position offset within the byte buffer at which the current aggregate value is stored - * @param time the time to be updated in the buffer as the last time - */ - void updateTimeWithNull(ByteBuffer buf, int position, long time) - { - buf.putLong(position, time); - buf.put(position + NULL_OFFSET, NullHandling.IS_NULL_BYTE); - } - - /** - *Abstract function which needs to be overridden by subclasses to set the initial value - */ - abstract void initValue(ByteBuffer buf, int position); - - /** - * Abstract function which needs to be overridden by subclasses to set the - * latest value in the buffer depending on the datatype - */ - abstract void putValue(ByteBuffer buf, int position, int index); - - @Override - public void close() - { - // no resources to cleanup - } -} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java deleted file mode 100644 index 119e13464a09..000000000000 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/SingleStringFirstDimensionVectorAggregator.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation.first; - -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; -import org.apache.druid.segment.vector.VectorValueSelector; - -import javax.annotation.Nullable; -import java.nio.ByteBuffer; - -public class SingleStringFirstDimensionVectorAggregator implements VectorAggregator -{ - private final VectorValueSelector timeSelector; - private final SingleValueDimensionVectorSelector valueDimensionVectorSelector; - private long firstTime; - private final int maxStringBytes; - private final boolean useDefault = NullHandling.replaceWithDefault(); - - public SingleStringFirstDimensionVectorAggregator( - VectorValueSelector timeSelector, - SingleValueDimensionVectorSelector valueDimensionVectorSelector, - int maxStringBytes - ) - { - this.timeSelector = timeSelector; - this.valueDimensionVectorSelector = valueDimensionVectorSelector; - this.maxStringBytes = maxStringBytes; - this.firstTime = Long.MAX_VALUE; - } - - @Override - public void init(ByteBuffer buf, int position) - { - buf.putLong(position, Long.MAX_VALUE); - buf.put( - position + NumericFirstVectorAggregator.NULL_OFFSET, - useDefault ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE - ); - buf.putInt(position + NumericFirstVectorAggregator.VALUE_OFFSET, 0); - } - - @Override - public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) - { - final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final int[] valueVector = valueDimensionVectorSelector.getRowVector(); - firstTime = buf.getLong(position); - int index; - - long earliestTime; - for (index = startRow; index < endRow; index++) { - if (nullTimeVector != null && nullTimeVector[index]) { - continue; - } - earliestTime = timeVector[index]; - if (earliestTime < firstTime) { - firstTime = earliestTime; - buf.putLong(position, firstTime); - buf.put(position + NumericFirstVectorAggregator.NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - buf.putInt(position + NumericFirstVectorAggregator.VALUE_OFFSET, valueVector[index]); - } - } - } - - @Override - public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) - { - final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final int[] values = valueDimensionVectorSelector.getRowVector(); - for (int i = 0; i < numRows; i++) { - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - int position = positions[i] + positionOffset; - int row = rows == null ? i : rows[i]; - long firstTime = buf.getLong(position); - if (timeVector[row] < firstTime) { - firstTime = timeVector[row]; - buf.putLong(position, firstTime); - buf.put(position + NumericFirstVectorAggregator.NULL_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - buf.putInt(position + NumericFirstVectorAggregator.VALUE_OFFSET, values[row]); - } - } - - } - - @Nullable - @Override - public Object get(ByteBuffer buf, int position) - { - int index = buf.getInt(position + NumericFirstVectorAggregator.VALUE_OFFSET); - long earliest = buf.getLong(position); - String strValue = valueDimensionVectorSelector.lookupName(index); - return new SerializablePairLongString(earliest, StringUtils.chop(strValue, maxStringBytes)); - } - - @Override - public void close() - { - // nothing to close - } -} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java deleted file mode 100644 index e4704e944ca3..000000000000 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregator.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation.first; - -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.VectorAggregator; -import org.apache.druid.segment.DimensionHandlerUtils; -import org.apache.druid.segment.vector.VectorObjectSelector; -import org.apache.druid.segment.vector.VectorValueSelector; - -import javax.annotation.Nullable; -import java.nio.ByteBuffer; - -public class StringFirstVectorAggregator implements VectorAggregator -{ - private static final SerializablePairLongString INIT = new SerializablePairLongString( - DateTimes.MAX.getMillis(), - null - ); - private final VectorValueSelector timeSelector; - private final VectorObjectSelector valueSelector; - private final int maxStringBytes; - - - public StringFirstVectorAggregator( - VectorValueSelector timeSelector, - VectorObjectSelector valueSelector, - int maxStringBytes - ) - { - this.timeSelector = timeSelector; - this.valueSelector = valueSelector; - this.maxStringBytes = maxStringBytes; - } - - @Override - public void init(ByteBuffer buf, int position) - { - StringFirstLastUtils.writePair(buf, position, INIT, maxStringBytes); - } - - @Override - public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) - { - if (timeSelector == null) { - return; - } - final long[] times = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final Object[] objectsWhichMightBeStrings = valueSelector.getObjectVector(); - long firstTime = buf.getLong(position); - int index; - for (int i = startRow; i < endRow; i++) { - if (times[i] > firstTime) { - continue; - } - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - index = i; - final boolean foldNeeded = FirstLastUtils.objectNeedsFoldCheck(objectsWhichMightBeStrings[index], SerializablePairLongString.class); - if (foldNeeded) { - final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( - null, - null, - null, - 0 -// timeSelector, -// valueSelector, -// index - ); - if (inPair != null) { - firstTime = buf.getLong(position); - if (inPair.lhs < firstTime) { - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(inPair.lhs, inPair.rhs), - maxStringBytes - ); - } - } - } else { - final long time = times[index]; - if (time < firstTime) { - final String value = DimensionHandlerUtils.convertObjectToString(objectsWhichMightBeStrings[index]); - firstTime = time; - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(time, value), - maxStringBytes - ); - } - } - } - - } - - @Override - public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) - { - final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final Object[] objectsWhichMightBeStrings = valueSelector.getObjectVector(); - - // iterate once over the object vector to find first non null element and - // determine if the type is Pair or not - boolean foldNeeded = false; - for (Object obj : objectsWhichMightBeStrings) { - if (obj == null) { - continue; - } else { - foldNeeded = FirstLastUtils.objectNeedsFoldCheck(obj, SerializablePairLongString.class); - break; - } - } - - for (int i = 0; i < numRows; i++) { - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - int position = positions[i] + positionOffset; - int row = rows == null ? i : rows[i]; - long firstTime = buf.getLong(position); - if (timeVector[row] < firstTime) { - if (foldNeeded) { - final SerializablePairLongString inPair = StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( - null, null, null, 0 -// timeSelector, -// valueSelector, -// row - ); - if (inPair != null) { - if (inPair.lhs < firstTime) { - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(inPair.lhs, inPair.rhs), - maxStringBytes - ); - } - } - } else { - final String value = DimensionHandlerUtils.convertObjectToString(objectsWhichMightBeStrings[row]); - firstTime = timeVector[row]; - StringFirstLastUtils.writePair( - buf, - position, - new SerializablePairLongString(firstTime, value), - maxStringBytes - ); - } - } - } - - } - - @Nullable - @Override - public Object get(ByteBuffer buf, int position) - { - return StringFirstLastUtils.readPair(buf, position); - } - - @Override - public void close() - { - // nothing to close - } -} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java similarity index 67% rename from processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java index 18537d668762..cd30e4cc69ee 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java @@ -17,11 +17,10 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.SerializablePairLongDouble; -import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -31,23 +30,34 @@ /** * Vectorized version of on heap 'last' aggregator for column selectors with type DOUBLE. */ -public class DoubleLastVectorAggregator extends NumericLastVectorAggregator +public class DoubleFirstLastVectorAggregator extends FirstLastVectorAggregator { - public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) + protected DoubleFirstLastVectorAggregator( + VectorValueSelector timeSelector, + VectorObjectSelector objectSelector, + SelectionPredicate selectionPredicate + ) { - super(timeSelector, null, objectSelector); + super(timeSelector, null, objectSelector, selectionPredicate); } - public DoubleLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + protected DoubleFirstLastVectorAggregator( + VectorValueSelector timeSelector, + VectorValueSelector valueSelector, + SelectionPredicate selectionPredicate + ) { - super(timeSelector, valueSelector, null); + super(timeSelector, valueSelector, null, selectionPredicate); } @Override public void init(ByteBuffer buf, int position) { buf.putLong(position, Long.MIN_VALUE); - buf.put(position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); + buf.put( + position + NULLITY_OFFSET, + NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE + ); buf.putDouble(position + VALUE_OFFSET, 0.0D); } @@ -56,14 +66,14 @@ public void init(ByteBuffer buf, int position) public Object get(ByteBuffer buf, int position) { long time = buf.getLong(position); - if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + if (buf.get(position + NULLITY_OFFSET) == NullHandling.IS_NULL_BYTE) { return new SerializablePairLongDouble(time, null); } return new SerializablePairLongDouble(time, buf.getDouble(position + VALUE_OFFSET)); } @Override - void putValue(ByteBuffer buf, int position, long time, Double value) + protected void putValue(ByteBuffer buf, int position, long time, Double value) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -71,7 +81,7 @@ void putValue(ByteBuffer buf, int position, long time, Double value) } @Override - void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + protected void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -79,7 +89,7 @@ void putValue(ByteBuffer buf, int position, long time, VectorValueSelector value } @Override - void putNull(ByteBuffer buf, int position, long time) + protected void putNull(ByteBuffer buf, int position, long time) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); @@ -88,7 +98,7 @@ void putNull(ByteBuffer buf, int position, long time) } @Override - void putDefaultValue(ByteBuffer buf, int position, long time) + protected void putDefaultValue(ByteBuffer buf, int position, long time) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -96,7 +106,7 @@ void putDefaultValue(ByteBuffer buf, int position, long time) } @Override - SerializablePairLongDouble readPairFromVectorSelectors( + protected SerializablePairLongDouble readPairFromVectorSelectors( boolean[] timeNullityVector, long[] timeVector, Object[] maybeFoldedObjects, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java similarity index 99% rename from processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java index f3225a579925..9b3bf694b37c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.SerializablePairLongFloat; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java similarity index 70% rename from processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java index baf096821c9a..69b8cbde636c 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/NumericLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast; import com.google.common.base.Preconditions; import org.apache.druid.collections.SerializablePair; @@ -32,13 +32,11 @@ /** * Base type for vectorized version of on heap 'last' aggregator for primitive numeric column selectors.. */ -public abstract class NumericLastVectorAggregator> +public abstract class FirstLastVectorAggregator> implements VectorAggregator { - protected static final byte IS_NULL_BYTE = (byte) 0; - protected static final byte IS_NOT_NULL_BYTE = (byte) 1; - protected static final int NULLITY_OFFSET = Long.BYTES; - protected static final int VALUE_OFFSET = NULLITY_OFFSET + Byte.BYTES; + public static final int NULLITY_OFFSET = Long.BYTES; + public static final int VALUE_OFFSET = NULLITY_OFFSET + Byte.BYTES; @Nullable private final VectorValueSelector timeSelector; @@ -46,6 +44,7 @@ public abstract class NumericLastVectorAggregator= selectedPair.lhs) { + } else if (selectionPredicate.apply(pair.lhs, selectedPair.lhs)) { selectedPair = pair; } } @@ -111,7 +104,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) // Something's been selected of the row vector if (selectedPair != null) { // Compare the latest value of the folded up row vector to the latest value in the buffer - if (selectedPair.lhs >= buf.getLong(position)) { + if (selectionPredicate.apply(selectedPair.lhs, buf.getLong(position))) { if (selectedPair.rhs != null) { putValue(buf, position, selectedPair.lhs, selectedPair.rhs); } else if (useDefault) { @@ -134,22 +127,21 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) continue; } // Find the latest time inside the vector objects - if (timeVector[selectedIndex] >= timeVector[index]) { + if (selectionPredicate.apply(timeVector[index], timeVector[selectedIndex])) { selectedIndex = index; } - - // Compare the selectedIndex's value to the value on the buffer. This way, we write to the buffer only once - // Weeds out empty vectors, where endRow == startRow - if (selectedIndex >= startRow) { - if (timeVector[selectedIndex] >= buf.getLong(position)) { - // Write the value here - if (valueNullityVector != null && !valueNullityVector[selectedIndex]) { - putValue(buf, position, timeVector[selectedIndex], valueSelector, selectedIndex); - } else if (useDefault) { - putDefaultValue(buf, position, timeVector[selectedIndex]); - } else { - putNull(buf, position, timeVector[index]); - } + } + // Compare the selectedIndex's value to the value on the buffer. This way, we write to the buffer only once + // Weeds out empty vectors, where endRow == startRow + if (selectedIndex >= startRow) { + if (selectionPredicate.apply(timeVector[selectedIndex], buf.getLong(position))) { + // Write the value here + if (valueNullityVector == null || !valueNullityVector[selectedIndex]) { + putValue(buf, position, timeVector[selectedIndex], valueSelector, selectedIndex); + } else if (useDefault) { + putDefaultValue(buf, position, timeVector[selectedIndex]); + } else { + putNull(buf, position, timeVector[selectedIndex]); } } } @@ -187,7 +179,7 @@ public void aggregate( // casting, and everything........ PairType pair = readPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, row); if (pair != null) { - if (pair.lhs >= lastTime) { + if (selectionPredicate.apply(pair.lhs, lastTime)) { if (pair.rhs != null) { putValue(buf, position, pair.lhs, pair.rhs); } else if (useDefault) { @@ -212,8 +204,8 @@ public void aggregate( // Don't aggregate values where time isn't present continue; } - if (timeVector[row] >= lastTime) { - if (valueNullityVector != null && !valueNullityVector[row]) { + if (selectionPredicate.apply(timeVector[row], lastTime)) { + if (valueNullityVector == null || !valueNullityVector[row]) { putValue(buf, position, timeVector[row], valueSelector, row); } else if (useDefault) { putDefaultValue(buf, position, timeVector[row]); @@ -224,43 +216,6 @@ public void aggregate( } } } -// -// @Nullable -// @Override -// public Object get(ByteBuffer buf, int position) -// { -// if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { -// return createPair(buf.getLong(position), null); -// } -// return createPair(buf.getLong(position), getValue(buf, position + VALUE_OFFSET)); -// } - -// private void updateTimeAndValue(ByteBuffer buf, int position, long time, RhsType value) -// { -// buf.putLong(position, time); -// buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); -// putValue(buf, position + VALUE_OFFSET, value); -// } -// -// private void updateTimeAndValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) -// { -// buf.putLong(position, time); -// buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); -// putValue(buf, position + VALUE_OFFSET, valueSelector, index); -// } -// -// private void updateTimeAndDefaultValue(ByteBuffer buf, int position, long time) -// { -// buf.putLong(position, time); -// buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); -// putDefaultValue(buf, position + VALUE_OFFSET); -// } -// -// private void updateTimeAndNull(ByteBuffer buf, int position, long time) -// { -// buf.putLong(position, time); -// buf.put(position + NULLITY_OFFSET, IS_NULL_BYTE); -// } /** * Sets the value at the position. Subclasses can assume that the value isn't null @@ -270,7 +225,7 @@ public void aggregate( * * It is only used if objectSelector is supplied */ - abstract void putValue(ByteBuffer buf, int position, long time, RhsType value); + protected abstract void putValue(ByteBuffer buf, int position, long time, RhsType value); /** * Sets the value represented by the valueSelector at the given index. A slightly redundant method to {@link #putValue(ByteBuffer, int, long, Object)} @@ -279,7 +234,7 @@ public void aggregate( * * It is used if valueSelector is supplied */ - abstract void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index); + protected abstract void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index); /** * Sets the default value for the type in the byte buffer at the given position. It is only used when replaceNullWithDefault = true, @@ -288,27 +243,17 @@ public void aggregate( * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of * the serialized pair) */ - abstract void putDefaultValue(ByteBuffer buf, int position, long time); + protected abstract void putDefaultValue(ByteBuffer buf, int position, long time); - abstract void putNull(ByteBuffer buf, int position, long time); + protected abstract void putNull(ByteBuffer buf, int position, long time); -// /** -// * Gets the value at the given position. It is okay to box, because we'd be wrapping it in a SerializablePair -// * -// * 'position' refers to the location where the value of the pair will get updated (as opposed to the beginning of -// * the serialized pair) -// */ -// abstract RhsType getValue(ByteBuffer buf, int position); - - abstract PairType readPairFromVectorSelectors( + protected abstract PairType readPairFromVectorSelectors( boolean[] timeNullityVector, long[] timeVector, Object[] maybeFoldedObjects, int index ); -// abstract PairType createPair(long time, @Nullable RhsType value); - @Override public void close() { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java similarity index 67% rename from processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java index e327db4d5cc4..cf57c9d72bb4 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java @@ -17,11 +17,10 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.SerializablePairLongFloat; -import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -31,24 +30,35 @@ /** * Vectorized version of on heap 'last' aggregator for column selectors with type FLOAT.. */ -public class FloatLastVectorAggregator extends NumericLastVectorAggregator +public class FloatFirstLastVectorAggregator extends FirstLastVectorAggregator { - public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) + protected FloatFirstLastVectorAggregator( + VectorValueSelector timeSelector, + VectorObjectSelector objectSelector, + SelectionPredicate selectionPredicate + ) { - super(timeSelector, null, objectSelector); + super(timeSelector, null, objectSelector, selectionPredicate); } - public FloatLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + protected FloatFirstLastVectorAggregator( + VectorValueSelector timeSelector, + VectorValueSelector valueSelector, + SelectionPredicate selectionPredicate + ) { - super(timeSelector, valueSelector, null); + super(timeSelector, valueSelector, null, selectionPredicate); } @Override public void init(ByteBuffer buf, int position) { buf.putLong(position, Long.MIN_VALUE); - buf.put(position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); + buf.put( + position + NULLITY_OFFSET, + NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE + ); buf.putFloat(position + VALUE_OFFSET, 0.0F); } @@ -57,14 +67,14 @@ public void init(ByteBuffer buf, int position) public Object get(ByteBuffer buf, int position) { long time = buf.getLong(position); - if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + if (buf.get(position + NULLITY_OFFSET) == NullHandling.IS_NULL_BYTE) { return new SerializablePairLongFloat(time, null); } return new SerializablePairLongFloat(time, buf.getFloat(position + VALUE_OFFSET)); } @Override - void putValue(ByteBuffer buf, int position, long time, Float value) + protected void putValue(ByteBuffer buf, int position, long time, Float value) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -72,7 +82,7 @@ void putValue(ByteBuffer buf, int position, long time, Float value) } @Override - void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + protected void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -80,7 +90,7 @@ void putValue(ByteBuffer buf, int position, long time, VectorValueSelector value } @Override - void putNull(ByteBuffer buf, int position, long time) + protected void putNull(ByteBuffer buf, int position, long time) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); @@ -88,7 +98,7 @@ void putNull(ByteBuffer buf, int position, long time) } @Override - void putDefaultValue(ByteBuffer buf, int position, long time) + protected void putDefaultValue(ByteBuffer buf, int position, long time) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -96,7 +106,7 @@ void putDefaultValue(ByteBuffer buf, int position, long time) } @Override - SerializablePairLongFloat readPairFromVectorSelectors( + protected SerializablePairLongFloat readPairFromVectorSelectors( boolean[] timeNullityVector, long[] timeVector, Object[] maybeFoldedObjects, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java similarity index 67% rename from processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java index 70943b1d4419..2695b176dad9 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java @@ -17,11 +17,10 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.SerializablePairLongLong; -import org.apache.druid.query.aggregation.first.FirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; @@ -31,23 +30,34 @@ /** * Vectorized version of on heap 'last' aggregator for column selectors with type LONG.. */ -public class LongLastVectorAggregator extends NumericLastVectorAggregator +public class LongFirstLastVectorAggregator extends FirstLastVectorAggregator { - public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorObjectSelector objectSelector) + protected LongFirstLastVectorAggregator( + VectorValueSelector timeSelector, + VectorObjectSelector objectSelector, + SelectionPredicate selectionPredicate + ) { - super(timeSelector, null, objectSelector); + super(timeSelector, null, objectSelector, selectionPredicate); } - public LongLastVectorAggregator(VectorValueSelector timeSelector, VectorValueSelector valueSelector) + protected LongFirstLastVectorAggregator( + VectorValueSelector timeSelector, + VectorValueSelector valueSelector, + SelectionPredicate selectionPredicate + ) { - super(timeSelector, valueSelector, null); + super(timeSelector, valueSelector, null, selectionPredicate); } @Override public void init(ByteBuffer buf, int position) { buf.putLong(position, Long.MIN_VALUE); - buf.put(position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? IS_NOT_NULL_BYTE : IS_NULL_BYTE); + buf.put( + position + NULLITY_OFFSET, + NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE + ); buf.putLong(position + VALUE_OFFSET, 0L); } @@ -56,14 +66,14 @@ public void init(ByteBuffer buf, int position) public Object get(ByteBuffer buf, int position) { long time = buf.getLong(position); - if (buf.get(position + NULLITY_OFFSET) == IS_NULL_BYTE) { + if (buf.get(position + NULLITY_OFFSET) == NullHandling.IS_NULL_BYTE) { return new SerializablePairLongLong(time, null); } return new SerializablePairLongLong(time, buf.getLong(position + VALUE_OFFSET)); } @Override - void putValue(ByteBuffer buf, int position, long time, Long value) + protected void putValue(ByteBuffer buf, int position, long time, Long value) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -71,7 +81,7 @@ void putValue(ByteBuffer buf, int position, long time, Long value) } @Override - void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + protected void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -79,7 +89,7 @@ void putValue(ByteBuffer buf, int position, long time, VectorValueSelector value } @Override - void putNull(ByteBuffer buf, int position, long time) + protected void putNull(ByteBuffer buf, int position, long time) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); @@ -87,7 +97,7 @@ void putNull(ByteBuffer buf, int position, long time) } @Override - void putDefaultValue(ByteBuffer buf, int position, long time) + protected void putDefaultValue(ByteBuffer buf, int position, long time) { buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); @@ -95,7 +105,7 @@ void putDefaultValue(ByteBuffer buf, int position, long time) } @Override - SerializablePairLongLong readPairFromVectorSelectors( + protected SerializablePairLongLong readPairFromVectorSelectors( boolean[] timeNullityVector, long[] timeVector, Object[] maybeFoldedObjects, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java new file mode 100644 index 000000000000..c3299cb60cc2 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.firstlast; + +public interface SelectionPredicate +{ + boolean apply(long currentTime, long selectedTime); + + SelectionPredicate FIRST_PREDICATE = (currentTime, selectedTime) -> currentTime <= selectedTime; + SelectionPredicate LAST_PREDICATE = (currentTime, selectedTime) -> currentTime >= selectedTime; +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java new file mode 100644 index 000000000000..f1072b894b5e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.firstlast; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.aggregation.SerializablePairLongString; +import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +public class SingleStringFirstLastDimensionVectorAggregator + extends FirstLastVectorAggregator +{ + private final SingleValueDimensionVectorSelector singleValueDimensionVectorSelector; + private final int maxStringBytes; + + protected SingleStringFirstLastDimensionVectorAggregator( + VectorValueSelector timeSelector, + SingleValueDimensionVectorSelector singleValueDimensionVectorSelector, + int maxStringBytes, + SelectionPredicate selectionPredicate + ) + { + super( + timeSelector, + new SingleValueDimensionVectorSelectorAdapter(singleValueDimensionVectorSelector), + null, + selectionPredicate + ); + this.singleValueDimensionVectorSelector = singleValueDimensionVectorSelector; + this.maxStringBytes = maxStringBytes; + } + + @Override + public void init(ByteBuffer buf, int position) + { + buf.putLong(position, Long.MIN_VALUE); + buf.put( + position + NULLITY_OFFSET, + NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE + ); + buf.putInt(position + VALUE_OFFSET, 0); + } + + /** + * Doesnt need to handle the null handling because that is taken care of while inserting the value, therefore in + * replaceWithDefault mode, it is always going to be non-null + * + * @param buf + * @param position + * @return + */ + @Nullable + @Override + public Object get(ByteBuffer buf, int position) + { + long time = buf.getLong(position); + if (buf.get(position + NULLITY_OFFSET) == NullHandling.IS_NULL_BYTE) { + return new SerializablePairLongString(time, null); + } + int index = buf.getInt(position + VALUE_OFFSET); + String value = singleValueDimensionVectorSelector.lookupName(index); + return new SerializablePairLongString(time, StringUtils.chop(value, maxStringBytes)); + } + + @Override + protected void putValue(ByteBuffer buf, int position, long time, String value) + { + throw DruidException.defensive("This method is not applicable to the SingleStringFirstLastDimensionVectorAggregator"); + } + + @Override + protected void putValue(ByteBuffer buf, int position, long time, VectorValueSelector valueSelector, int index) + { + buf.putLong(position, time); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putInt( + position + VALUE_OFFSET, + ((SingleValueDimensionVectorSelectorAdapter) valueSelector).singleValueDimensionVectorSelector.getRowVector()[index] + ); + } + + @Override + protected void putDefaultValue(ByteBuffer buf, int position, long time) + { + buf.putLong(position, Long.MIN_VALUE); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); + buf.putInt(position + VALUE_OFFSET, 0); + } + + @Override + protected void putNull(ByteBuffer buf, int position, long time) + { + buf.putLong(position, Long.MIN_VALUE); + buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); + buf.putInt(position + VALUE_OFFSET, 0); + } + + @Override + protected SerializablePairLongString readPairFromVectorSelectors( + boolean[] timeNullityVector, + long[] timeVector, + Object[] maybeFoldedObjects, + int index + ) + { + throw DruidException.defensive("This method is not applicable to the SingleStringFirstLastDimensionVectorAggregator"); + } + + /** + * TODO(laksh): We use this adapter class because.. + */ + private static class SingleValueDimensionVectorSelectorAdapter implements VectorValueSelector + { + + private final SingleValueDimensionVectorSelector singleValueDimensionVectorSelector; + + public SingleValueDimensionVectorSelectorAdapter(SingleValueDimensionVectorSelector singleValueDimensionVectorSelector) + { + this.singleValueDimensionVectorSelector = singleValueDimensionVectorSelector; + } + + @Override + public int getMaxVectorSize() + { + throw DruidException.defensive("Unexpected call"); + } + + @Override + public int getCurrentVectorSize() + { + throw DruidException.defensive("Unexpected call"); + } + + @Override + public long[] getLongVector() + { + throw DruidException.defensive("Unexpected call"); + } + + @Override + public float[] getFloatVector() + { + throw DruidException.defensive("Unexpected call"); + } + + @Override + public double[] getDoubleVector() + { + throw DruidException.defensive("Unexpected call"); + } + + /** + * TODO(laksh): Only useful method + */ + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + + public SingleValueDimensionVectorSelector getSingleValueDimensionVectorSelector() + { + return singleValueDimensionVectorSelector; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastUtils.java similarity index 96% rename from processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastUtils.java index db0e812aca8e..afbb94faf8f3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/first/StringFirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastUtils.java @@ -17,15 +17,13 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.DimensionHandlerUtils; -import org.apache.druid.segment.vector.VectorObjectSelector; -import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java similarity index 54% rename from processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java rename to processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java index 0bd4a6e2d491..ccc65081210a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java @@ -1,16 +1,34 @@ -package org.apache.druid.query.aggregation.last; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.firstlast; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.first.StringFirstLastUtils; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.nio.ByteBuffer; -public class StringLastVectorAggregator extends NumericLastVectorAggregator +public class StringFirstLastVectorAggregator extends FirstLastVectorAggregator { // Initialized with MIN_VALUE instead of DateTimes.MIN.getMillis(), as it can be a custom timeSelector (provided via LATEST_BY) // that has a lower min than the times.min @@ -21,13 +39,14 @@ public class StringLastVectorAggregator extends NumericLastVectorAggregator= startRow; index--) { - if (nullTimeVector != null && nullTimeVector[index]) { - continue; - } - latestTime = timeVector[index]; - if (latestTime > lastTime) { - lastTime = latestTime; - buf.putLong(position, lastTime); - buf.put(position + NumericLastVectorAggregator.NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - buf.putInt(position + NumericLastVectorAggregator.VALUE_OFFSET, valueVector[index]); - } - } - } - - @Override - public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) - { - final long[] timeVector = timeSelector.getLongVector(); - final boolean[] nullTimeVector = timeSelector.getNullVector(); - final int[] values = valueDimensionVectorSelector.getRowVector(); - for (int i = numRows - 1; i >= 0; i--) { - if (nullTimeVector != null && nullTimeVector[i]) { - continue; - } - int position = positions[i] + positionOffset; - int row = rows == null ? i : rows[i]; - lastTime = buf.getLong(position); - if (timeVector[row] > lastTime) { - lastTime = timeVector[row]; - buf.putLong(position, lastTime); - buf.put(position + NumericLastVectorAggregator.NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); - buf.putInt(position + NumericLastVectorAggregator.VALUE_OFFSET, values[row]); - } - } - } - - @Nullable - @Override - public Object get(ByteBuffer buf, int position) - { - int index = buf.getInt(position + NumericLastVectorAggregator.VALUE_OFFSET); - long earliest = buf.getLong(position); - String strValue = valueDimensionVectorSelector.lookupName(index); - return new SerializablePairLongString(earliest, StringUtils.chop(strValue, maxStringBytes)); - } - - @Override - public void close() - { - // nothing to close - } -} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java index 44c998cd89cd..ff7b18c3dca2 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregatorFactoryTest.java @@ -28,15 +28,15 @@ import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.StringAnyAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.mean.DoubleMeanAggregatorFactory; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.query.dimension.DefaultDimensionSpec; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstAggregationTest.java index 7109a0a47961..1d521d1a5c95 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java similarity index 96% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java index e055930a2169..9ae9ee0b1dbd 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/DoubleFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -232,9 +233,9 @@ public void testFactory() @Test public void initValueShouldInitZero() { - target.initValue(buf, 0); - double initVal = buf.getDouble(0); - Assert.assertEquals(0, initVal, EPSILON); + target.init(buf, 0); + Assert.assertEquals(Long.MIN_VALUE, buf.getLong(0)); + Assert.assertEquals(0, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstAggregationTest.java index 0df142677437..e1d8ee4c7409 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java similarity index 97% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java index a95e89bd32af..92aab9d696b1 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/FloatFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java @@ -17,12 +17,13 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -263,9 +264,9 @@ public void testFactory() @Test public void initValueShouldBeZero() { - target.initValue(buf, 0); - float initVal = buf.getFloat(0); - Assert.assertEquals(0.0f, initVal, EPSILON); + target.init(buf, 0); + Assert.assertEquals(Long.MIN_VALUE, buf.getLong(0)); + Assert.assertEquals(0.0f, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstAggregationTest.java index 104dbcfa60dd..e11c27c38c77 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java similarity index 96% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java index a45e0f25563d..cebae676425e 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/LongFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java @@ -17,12 +17,13 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -236,9 +237,9 @@ public void testFactory() @Test public void initValueShouldInitZero() { - target.initValue(buf, 0); - long initVal = buf.getLong(0); - Assert.assertEquals(0, initVal); + target.init(buf, 0); + Assert.assertEquals(Long.MIN_VALUE, buf.getLong(0)); + Assert.assertEquals(0, buf.getLong(FirstLastVectorAggregator.VALUE_OFFSET)); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstAggregationTest.java index d2f332f0bb85..6f6bd41f6bc1 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstBufferAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstBufferAggregatorTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstBufferAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstBufferAggregatorTest.java index 2653940ae389..9f72ff8fd313 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstBufferAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstBufferAggregatorTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongString; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstLastUtilsTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstLastUtilsTest.java similarity index 94% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstLastUtilsTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstLastUtilsTest.java index b4e4088535cb..d6b95a89a995 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstLastUtilsTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstLastUtilsTest.java @@ -17,10 +17,11 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.query.aggregation.SerializablePairLongString; +import org.apache.druid.query.aggregation.firstlast.StringFirstLastUtils; import org.junit.Assert; import org.junit.Test; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstTimeseriesQueryTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java index 267fa52ab639..2e960e6186e3 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstTimeseriesQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstTimeseriesQueryTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java index e3b461da5687..516c28f015f6 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/first/StringFirstVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.first; +package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastAggregationTest.java index bbcf9be37a2b..1135fb76ca6f 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java index 93ad62b6c94f..1e91e16ce269 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/DoubleLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongDouble; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastAggregationTest.java index 7030f6671e1b..77ac1083f454 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java similarity index 97% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java index b522362bdaa1..589ac297f173 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/FloatLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java @@ -17,12 +17,13 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FloatFirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -69,7 +70,7 @@ public class FloatLastVectorAggregatorTest extends InitializedNullHandlingTest private VectorObjectSelector selector; private BaseLongVectorValueSelector timeSelector; private ByteBuffer buf; - private FloatLastVectorAggregator target; + private FloatFirstLastVectorAggregator target; private FloatLastAggregatorFactory floatLastAggregatorFactory; private VectorColumnSelectorFactory selectorFactory; @@ -231,7 +232,7 @@ public void testFactory() Assert.assertTrue(floatLastAggregatorFactory.canVectorize(selectorFactory)); VectorAggregator vectorAggregator = floatLastAggregatorFactory.factorizeVector(selectorFactory); Assert.assertNotNull(vectorAggregator); - Assert.assertEquals(FloatLastVectorAggregator.class, vectorAggregator.getClass()); + Assert.assertEquals(FloatFirstLastVectorAggregator.class, vectorAggregator.getClass()); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastAggregationTest.java index 111f86884251..6a922b63d5c6 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java similarity index 97% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java index 48b22f6a18b4..6eaa896558fc 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/LongLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java @@ -17,12 +17,13 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; @@ -238,11 +239,12 @@ public void testFactory() } @Test - public void initValueShouldInitZero() + public void initTest() { target.init(buf, 0); long initVal = buf.getLong(0); - Assert.assertEquals(0, initVal); + Assert.assertEquals(Long.MIN_VALUE, initVal); + Assert.assertEquals(0, buf.getLong(FirstLastVectorAggregator.VALUE_OFFSET)); } @Test diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastAggregationTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastAggregationTest.java index 6a6aa1e6509d..974d668b9818 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastAggregationTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Pair; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastBufferAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastBufferAggregatorTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastBufferAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastBufferAggregatorTest.java index 37a1c417e198..4c3d7fea39e8 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastBufferAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastBufferAggregatorTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.SerializablePairLongString; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastTimeseriesQueryTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java index 318a2a47fd68..6bbada405b93 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastTimeseriesQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastTimeseriesQueryTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java similarity index 99% rename from processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java index 50d7d63a4df8..861ab3d7487e 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/last/StringLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java @@ -17,9 +17,8 @@ * under the License. */ -package org.apache.druid.query.aggregation.last; +package org.apache.druid.query.aggregation.firstlast.last; -import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.VectorAggregator; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java index ceea41d427bc..f5a19119b84c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/post/FinalizingFieldAccessPostAggregatorTest.java @@ -34,7 +34,7 @@ import org.apache.druid.query.aggregation.CountAggregator; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; import org.apache.druid.query.groupby.GroupByQueryRunnerTest; import org.apache.druid.query.groupby.ResultRow; import org.apache.druid.query.timeseries.TimeseriesQuery; diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java index 726d04f65c86..6545446e0dee 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryQueryToolChestTest.java @@ -51,10 +51,10 @@ import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; import org.apache.druid.query.aggregation.post.ExpressionPostAggregator; import org.apache.druid.query.dimension.DefaultDimensionSpec; diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java index e3e19a172aef..cc5f0f996d40 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java @@ -76,10 +76,10 @@ import org.apache.druid.query.aggregation.LongMaxAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.mean.DoubleMeanAggregatorFactory; import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java index b672798b1ba6..f1afabe9c75e 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java @@ -36,7 +36,7 @@ import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.SerializablePairLongString; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java index aefad5c6dde9..8f0ffadef8a0 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -54,9 +54,9 @@ import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.expression.TestExprMacroTable; diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java index 23d202c72be0..f53fc77cb216 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryQueryToolChestTest.java @@ -48,11 +48,11 @@ import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregator; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.ConstantPostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; diff --git a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java index 4163afb997fa..79adf89fb86b 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/TopNQueryRunnerTest.java @@ -59,13 +59,13 @@ import org.apache.druid.query.aggregation.FloatMinAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.post.ExpressionPostAggregator; import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.dimension.DefaultDimensionSpec; diff --git a/processing/src/test/java/org/apache/druid/query/topn/UnnestTopNQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/topn/UnnestTopNQueryRunnerTest.java index ba0025490fa1..cb4ee3b942b1 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/UnnestTopNQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/UnnestTopNQueryRunnerTest.java @@ -38,7 +38,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory; import org.apache.druid.query.aggregation.DoubleMinAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.ordering.StringComparators; diff --git a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java index ddea2cbf62b0..c0b8cceb24b6 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexMergerRollupTest.java @@ -23,14 +23,14 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.segment.data.IncrementalIndexTest; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java index b47985ea95a9..73209a76a987 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/EarliestLatestAnySqlAggregator.java @@ -51,14 +51,14 @@ import org.apache.druid.query.aggregation.any.FloatAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.StringAnyAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index a48ce3df6b4a..4d4dd189078f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -64,16 +64,16 @@ import org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory; import org.apache.druid.query.aggregation.any.StringAnyAggregatorFactory; import org.apache.druid.query.aggregation.cardinality.CardinalityAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.FloatFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.FloatFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSimpleQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSimpleQueryTest.java index 6fe2205a7ae0..3d7d2a2de33e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSimpleQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSimpleQueryTest.java @@ -24,8 +24,8 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.first.StringFirstAggregatorFactory; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.StringFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.filter.LikeDimFilter; import org.apache.druid.query.groupby.GroupByQuery; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index 2303d51e02ab..068ac21c8606 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -48,12 +48,12 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.FloatSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.first.DoubleFirstAggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.DoubleFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.DoubleLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.FloatLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.LongLastAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; -import org.apache.druid.query.aggregation.last.DoubleLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.FloatLastAggregatorFactory; -import org.apache.druid.query.aggregation.last.LongLastAggregatorFactory; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; From 40a47b20ea09be513fcc8637b9ea63eeacf37467 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 4 Apr 2024 14:54:08 +0530 Subject: [PATCH 04/10] test fix --- .../DoubleFirstLastVectorAggregator.java | 8 +++-- .../aggregation/firstlast/FirstLastUtils.java | 13 -------- .../firstlast/FirstLastVectorAggregator.java | 22 +++++++++---- .../FloatFirstLastVectorAggregator.java | 7 ++-- .../LongFirstLastVectorAggregator.java | 6 +++- .../firstlast/SelectionPredicate.java | 33 +++++++++++++++++-- ...ingFirstLastDimensionVectorAggregator.java | 13 +++----- .../StringFirstLastVectorAggregator.java | 23 ++++++++----- .../DoubleFirstVectorAggregationTest.java | 2 +- .../FloatFirstVectorAggregationTest.java | 4 +-- .../first/LongFirstVectorAggregationTest.java | 2 +- 11 files changed, 85 insertions(+), 48 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java index cd30e4cc69ee..728018691550 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/DoubleFirstLastVectorAggregator.java @@ -28,10 +28,12 @@ import java.nio.ByteBuffer; /** - * Vectorized version of on heap 'last' aggregator for column selectors with type DOUBLE. + * Vectorized version of on heap aggregator for column selectors with type DOUBLE. */ public class DoubleFirstLastVectorAggregator extends FirstLastVectorAggregator { + private final SelectionPredicate selectionPredicate; + protected DoubleFirstLastVectorAggregator( VectorValueSelector timeSelector, VectorObjectSelector objectSelector, @@ -39,6 +41,7 @@ protected DoubleFirstLastVectorAggregator( ) { super(timeSelector, null, objectSelector, selectionPredicate); + this.selectionPredicate = selectionPredicate; } protected DoubleFirstLastVectorAggregator( @@ -48,12 +51,13 @@ protected DoubleFirstLastVectorAggregator( ) { super(timeSelector, valueSelector, null, selectionPredicate); + this.selectionPredicate = selectionPredicate; } @Override public void init(ByteBuffer buf, int position) { - buf.putLong(position, Long.MIN_VALUE); + buf.putLong(position, selectionPredicate.initValue()); buf.put( position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java index 9b3bf694b37c..460118be9278 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java @@ -58,19 +58,6 @@ public static boolean selectorNeedsFoldCheck( || pairClass.isAssignableFrom(clazz); } - /** - * Returns whether an object *might* is assignable to/from the pairClass. - */ - public static boolean objectNeedsFoldCheck(Object obj, Class pairClass) - { - if (obj == null) { - return false; - } - final Class clazz = obj.getClass(); - return clazz.isAssignableFrom(pairClass) - || pairClass.isAssignableFrom(clazz); - } - @Nullable public static SerializablePairLongDouble readDoublePairFromVectorSelectors( @Nullable boolean[] timeNullityVector, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java index 69b8cbde636c..f94f796782ba 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java @@ -63,10 +63,12 @@ public FirstLastVectorAggregator( SelectionPredicate selectionPredicate ) { - Preconditions.checkArgument( - (valueSelector != null && objectSelector == null) || (valueSelector == null && objectSelector != null), - "exactly one of 'valueSelector' and 'objectSelector' must be provided" - ); + if (timeSelector != null) { + Preconditions.checkArgument( + (valueSelector != null && objectSelector == null) || (valueSelector == null && objectSelector != null), + "exactly one of 'valueSelector' and 'objectSelector' must be provided" + ); + } this.timeSelector = timeSelector; this.valueSelector = valueSelector; this.objectSelector = objectSelector; @@ -82,6 +84,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) if (timeSelector == null) { return; } + // If objectSelector isn't null, then the objects might be folded up. If that's the case, whatever's represented by // the timeSelector doesn't hold any relevance. if (objectSelector != null) { @@ -120,20 +123,25 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) final boolean[] timeNullityVector = timeSelector.getNullVector(); final long[] timeVector = timeSelector.getLongVector(); final boolean[] valueNullityVector = valueSelector.getNullVector(); - int selectedIndex = endRow - 1; + Integer selectedIndex = null; + for (int index = endRow - 1; index >= startRow; --index) { if (timeNullityVector != null && timeNullityVector[index]) { // Don't aggregate values where time isn't present continue; } // Find the latest time inside the vector objects - if (selectionPredicate.apply(timeVector[index], timeVector[selectedIndex])) { + if (selectedIndex == null) { selectedIndex = index; + } else { + if (selectionPredicate.apply(timeVector[index], timeVector[selectedIndex])) { + selectedIndex = index; + } } } // Compare the selectedIndex's value to the value on the buffer. This way, we write to the buffer only once // Weeds out empty vectors, where endRow == startRow - if (selectedIndex >= startRow) { + if (selectedIndex != null) { if (selectionPredicate.apply(timeVector[selectedIndex], buf.getLong(position))) { // Write the value here if (valueNullityVector == null || !valueNullityVector[selectedIndex]) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java index cf57c9d72bb4..2edebc307823 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FloatFirstLastVectorAggregator.java @@ -28,10 +28,11 @@ import java.nio.ByteBuffer; /** - * Vectorized version of on heap 'last' aggregator for column selectors with type FLOAT.. + * Vectorized version of on heap 'last' aggregator for column selectors with type FLOAT. */ public class FloatFirstLastVectorAggregator extends FirstLastVectorAggregator { + private final SelectionPredicate selectionPredicate; protected FloatFirstLastVectorAggregator( VectorValueSelector timeSelector, @@ -40,6 +41,7 @@ protected FloatFirstLastVectorAggregator( ) { super(timeSelector, null, objectSelector, selectionPredicate); + this.selectionPredicate = selectionPredicate; } protected FloatFirstLastVectorAggregator( @@ -49,12 +51,13 @@ protected FloatFirstLastVectorAggregator( ) { super(timeSelector, valueSelector, null, selectionPredicate); + this.selectionPredicate = selectionPredicate; } @Override public void init(ByteBuffer buf, int position) { - buf.putLong(position, Long.MIN_VALUE); + buf.putLong(position, selectionPredicate.initValue()); buf.put( position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java index 2695b176dad9..9b77426dfee3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/LongFirstLastVectorAggregator.java @@ -32,6 +32,8 @@ */ public class LongFirstLastVectorAggregator extends FirstLastVectorAggregator { + private final SelectionPredicate selectionPredicate; + protected LongFirstLastVectorAggregator( VectorValueSelector timeSelector, VectorObjectSelector objectSelector, @@ -39,6 +41,7 @@ protected LongFirstLastVectorAggregator( ) { super(timeSelector, null, objectSelector, selectionPredicate); + this.selectionPredicate = selectionPredicate; } protected LongFirstLastVectorAggregator( @@ -48,12 +51,13 @@ protected LongFirstLastVectorAggregator( ) { super(timeSelector, valueSelector, null, selectionPredicate); + this.selectionPredicate = selectionPredicate; } @Override public void init(ByteBuffer buf, int position) { - buf.putLong(position, Long.MIN_VALUE); + buf.putLong(position, selectionPredicate.initValue()); buf.put( position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java index c3299cb60cc2..3f448ac1f1f1 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java @@ -19,10 +19,39 @@ package org.apache.druid.query.aggregation.firstlast; +// TODO(laksh): javadoc public interface SelectionPredicate { + long initValue(); boolean apply(long currentTime, long selectedTime); - SelectionPredicate FIRST_PREDICATE = (currentTime, selectedTime) -> currentTime <= selectedTime; - SelectionPredicate LAST_PREDICATE = (currentTime, selectedTime) -> currentTime >= selectedTime; + SelectionPredicate FIRST_PREDICATE = new SelectionPredicate() + { + @Override + public long initValue() + { + return Long.MAX_VALUE; + } + + @Override + public boolean apply(long currentTime, long selectedTime) + { + return currentTime <= selectedTime; + } + }; + + SelectionPredicate LAST_PREDICATE = new SelectionPredicate() + { + @Override + public long initValue() + { + return Long.MIN_VALUE; + } + + @Override + public boolean apply(long currentTime, long selectedTime) + { + return currentTime >= selectedTime; + } + }; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java index f1072b894b5e..7b1bbddec932 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java @@ -34,6 +34,7 @@ public class SingleStringFirstLastDimensionVectorAggregator { private final SingleValueDimensionVectorSelector singleValueDimensionVectorSelector; private final int maxStringBytes; + private final SelectionPredicate selectionPredicate; protected SingleStringFirstLastDimensionVectorAggregator( VectorValueSelector timeSelector, @@ -50,12 +51,13 @@ protected SingleStringFirstLastDimensionVectorAggregator( ); this.singleValueDimensionVectorSelector = singleValueDimensionVectorSelector; this.maxStringBytes = maxStringBytes; + this.selectionPredicate = selectionPredicate; } @Override public void init(ByteBuffer buf, int position) { - buf.putLong(position, Long.MIN_VALUE); + buf.putLong(position, selectionPredicate.initValue()); buf.put( position + NULLITY_OFFSET, NullHandling.replaceWithDefault() ? NullHandling.IS_NOT_NULL_BYTE : NullHandling.IS_NULL_BYTE @@ -104,7 +106,7 @@ protected void putValue(ByteBuffer buf, int position, long time, VectorValueSele @Override protected void putDefaultValue(ByteBuffer buf, int position, long time) { - buf.putLong(position, Long.MIN_VALUE); + buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NOT_NULL_BYTE); buf.putInt(position + VALUE_OFFSET, 0); } @@ -112,7 +114,7 @@ protected void putDefaultValue(ByteBuffer buf, int position, long time) @Override protected void putNull(ByteBuffer buf, int position, long time) { - buf.putLong(position, Long.MIN_VALUE); + buf.putLong(position, time); buf.put(position + NULLITY_OFFSET, NullHandling.IS_NULL_BYTE); buf.putInt(position + VALUE_OFFSET, 0); } @@ -180,10 +182,5 @@ public boolean[] getNullVector() { return null; } - - public SingleValueDimensionVectorSelector getSingleValueDimensionVectorSelector() - { - return singleValueDimensionVectorSelector; - } } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java index ccc65081210a..118d28e0c366 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/StringFirstLastVectorAggregator.java @@ -30,14 +30,8 @@ public class StringFirstLastVectorAggregator extends FirstLastVectorAggregator { - // Initialized with MIN_VALUE instead of DateTimes.MIN.getMillis(), as it can be a custom timeSelector (provided via LATEST_BY) - // that has a lower min than the times.min - private static final SerializablePairLongString INIT = new SerializablePairLongString( - Long.MIN_VALUE, - null - ); - private final int maxStringBytes; + private final SelectionPredicate selectionPredicate; protected StringFirstLastVectorAggregator( @Nullable final VectorValueSelector timeSelector, @@ -48,12 +42,18 @@ protected StringFirstLastVectorAggregator( { super(timeSelector, null, objectSelector, selectionPredicate); this.maxStringBytes = maxStringBytes; + this.selectionPredicate = selectionPredicate; } @Override public void init(ByteBuffer buf, int position) { - StringFirstLastUtils.writePair(buf, position, INIT, maxStringBytes); + StringFirstLastUtils.writePair( + buf, + position, + new SerializablePairLongString(selectionPredicate.initValue(), null), + maxStringBytes + ); } @Nullable @@ -100,6 +100,11 @@ protected SerializablePairLongString readPairFromVectorSelectors( int index ) { - return StringFirstLastUtils.readPairFromVectorSelectorsAtIndex(timeNullityVector, timeVector, maybeFoldedObjects, index); + return StringFirstLastUtils.readPairFromVectorSelectorsAtIndex( + timeNullityVector, + timeVector, + maybeFoldedObjects, + index + ); } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java index 9ae9ee0b1dbd..0a20f772c1cf 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java @@ -234,7 +234,7 @@ public void testFactory() public void initValueShouldInitZero() { target.init(buf, 0); - Assert.assertEquals(Long.MIN_VALUE, buf.getLong(0)); + Assert.assertEquals(Long.MAX_VALUE, buf.getLong(0)); Assert.assertEquals(0, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java index 92aab9d696b1..dc92164757c4 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java @@ -265,7 +265,7 @@ public void testFactory() public void initValueShouldBeZero() { target.init(buf, 0); - Assert.assertEquals(Long.MIN_VALUE, buf.getLong(0)); + Assert.assertEquals(Long.MAX_VALUE, buf.getLong(0)); Assert.assertEquals(0.0f, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @@ -285,7 +285,7 @@ public void aggregateNulls1() target1.init(buf, 0); target1.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target1.get(buf, 0); - Assert.assertEquals(Long.MAX_VALUE, result.lhs.longValue()); + Assert.assertEquals(pairs[0].lhs.longValue(), result.lhs.longValue()); Assert.assertEquals(NullHandling.defaultFloatValue(), result.rhs); } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java index cebae676425e..0ac8ff9275aa 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java @@ -238,7 +238,7 @@ public void testFactory() public void initValueShouldInitZero() { target.init(buf, 0); - Assert.assertEquals(Long.MIN_VALUE, buf.getLong(0)); + Assert.assertEquals(Long.MAX_VALUE, buf.getLong(0)); Assert.assertEquals(0, buf.getLong(FirstLastVectorAggregator.VALUE_OFFSET)); } From 7f4e6d373ff788d45ee20b2f1a500a4340d9e46e Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 5 Apr 2024 02:51:29 +0530 Subject: [PATCH 05/10] test fix --- .../firstlast/SelectionPredicate.java | 8 +++++--- .../druid/sql/calcite/CalciteQueryTest.java | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java index 3f448ac1f1f1..726d15bf3b4b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation.firstlast; +import org.apache.druid.java.util.common.DateTimes; + // TODO(laksh): javadoc public interface SelectionPredicate { @@ -30,13 +32,13 @@ public interface SelectionPredicate @Override public long initValue() { - return Long.MAX_VALUE; + return DateTimes.MAX.getMillis(); } @Override public boolean apply(long currentTime, long selectedTime) { - return currentTime <= selectedTime; + return currentTime < selectedTime; } }; @@ -45,7 +47,7 @@ public boolean apply(long currentTime, long selectedTime) @Override public long initValue() { - return Long.MIN_VALUE; + return DateTimes.MIN.getMillis(); } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 4d4dd189078f..0ab2136c0f2e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -712,7 +712,22 @@ public void testEarliestAggregators() .build() ), ImmutableList.of( - new Object[]{1L, 1.0f, "", "", 2L, 2.0f, "1", 1L, 3.0f, "2", 2L, 4.0f, "21", "21"} + new Object[]{ + 1L, + 1.0f, + "", + "", + 2L, + 2.0f, + "1", + 1L, + NullHandling.replaceWithDefault() ? 4.0f : 3.0f, + NullHandling.replaceWithDefault() ? "1" : "2", + 2L, + NullHandling.replaceWithDefault() ? 5.0f : 4.0f, + NullHandling.replaceWithDefault() ? "11" : "21", + NullHandling.replaceWithDefault() ? "11" : "21" + } ) ); } From 67e2dc2a1c7741b4b2e9af56b6951f7949c5091d Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 15 Apr 2024 13:47:39 +0530 Subject: [PATCH 06/10] tests fix --- .../firstlast/FirstLastVectorAggregator.java | 5 +++-- .../druid/sql/calcite/CalciteQueryTest.java | 17 +---------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java index f94f796782ba..8c21ab77275b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java @@ -94,7 +94,8 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) PairType selectedPair = null; - for (int index = endRow - 1; index >= startRow; --index) { + for (int index = startRow; index < endRow; ++index) { + PairType pair = readPairFromVectorSelectors(timeNullityVector, timeVector, maybeFoldedObjects, index); if (pair != null) { if (selectedPair == null) { @@ -125,7 +126,7 @@ public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) final boolean[] valueNullityVector = valueSelector.getNullVector(); Integer selectedIndex = null; - for (int index = endRow - 1; index >= startRow; --index) { + for (int index = startRow; index < endRow; ++index) { if (timeNullityVector != null && timeNullityVector[index]) { // Don't aggregate values where time isn't present continue; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 0ab2136c0f2e..4d4dd189078f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -712,22 +712,7 @@ public void testEarliestAggregators() .build() ), ImmutableList.of( - new Object[]{ - 1L, - 1.0f, - "", - "", - 2L, - 2.0f, - "1", - 1L, - NullHandling.replaceWithDefault() ? 4.0f : 3.0f, - NullHandling.replaceWithDefault() ? "1" : "2", - 2L, - NullHandling.replaceWithDefault() ? 5.0f : 4.0f, - NullHandling.replaceWithDefault() ? "11" : "21", - NullHandling.replaceWithDefault() ? "11" : "21" - } + new Object[]{1L, 1.0f, "", "", 2L, 2.0f, "1", 1L, 3.0f, "2", 2L, 4.0f, "21", "21"} ) ); } From e494239c2b5bedc000a2146266bfebbc84ac12e7 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Tue, 16 Apr 2024 10:07:37 +0530 Subject: [PATCH 07/10] tests fix --- .../firstlast/SelectionPredicate.java | 13 ++- ...ingFirstLastDimensionVectorAggregator.java | 11 ++- ...a => DoubleFirstVectorAggregatorTest.java} | 41 +++++++--- ...va => FloatFirstVectorAggregatorTest.java} | 17 ++-- ...ava => LongFirstVectorAggregatorTest.java} | 17 ++-- .../StringFirstVectorAggregatorTest.java | 13 +-- .../last/DoubleLastVectorAggregatorTest.java | 42 +++++++--- .../last/FloatLastVectorAggregatorTest.java | 18 ++-- .../last/LongLastVectorAggregatorTest.java | 82 ++++++++++++++++--- .../last/StringLastVectorAggregatorTest.java | 26 +++--- 10 files changed, 198 insertions(+), 82 deletions(-) rename processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/{DoubleFirstVectorAggregationTest.java => DoubleFirstVectorAggregatorTest.java} (88%) rename processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/{FloatFirstVectorAggregationTest.java => FloatFirstVectorAggregatorTest.java} (96%) rename processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/{LongFirstVectorAggregationTest.java => LongFirstVectorAggregatorTest.java} (95%) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java index 726d15bf3b4b..fd1ca915b473 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SelectionPredicate.java @@ -21,10 +21,21 @@ import org.apache.druid.java.util.common.DateTimes; -// TODO(laksh): javadoc +/** + * Differentiating factor between the first and the last aggregator. Specifies the + */ public interface SelectionPredicate { + /** + * @return Time value to initialize the aggregation with + */ long initValue(); + + /** + * @param currentTime Time of the current row + * @param selectedTime Aggregated time value + * @return true if the current row should be selected over the aggregated value, else false + */ boolean apply(long currentTime, long selectedTime); SelectionPredicate FIRST_PREDICATE = new SelectionPredicate() diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java index 7b1bbddec932..031140e4f6c5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/SingleStringFirstLastDimensionVectorAggregator.java @@ -68,10 +68,6 @@ public void init(ByteBuffer buf, int position) /** * Doesnt need to handle the null handling because that is taken care of while inserting the value, therefore in * replaceWithDefault mode, it is always going to be non-null - * - * @param buf - * @param position - * @return */ @Nullable @Override @@ -131,7 +127,10 @@ protected SerializablePairLongString readPairFromVectorSelectors( } /** - * TODO(laksh): We use this adapter class because.. + * Adapter class to from {@link SingleValueDimensionVectorSelector} to {@link VectorValueSelector}, to pass to the parent + * class. The parent class only uses the passed in selector for the null check, therefore {@link #getNullVector()} is + * the only relevant method implemented by the adapter. Each string value (even null) is assigned an id that will get stored, + * therefore fetching the nullVector returns null (i.e. no null values) */ private static class SingleValueDimensionVectorSelectorAdapter implements VectorValueSelector { @@ -174,7 +173,7 @@ public double[] getDoubleVector() } /** - * TODO(laksh): Only useful method + * This is the only useful method, that will get used by the parent class. */ @Nullable @Override diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregatorTest.java similarity index 88% rename from processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregatorTest.java index 0a20f772c1cf..734d7e3d6e8c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/DoubleFirstVectorAggregatorTest.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.firstlast.first; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.VectorAggregator; @@ -27,6 +28,7 @@ import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseDoubleVectorValueSelector; import org.apache.druid.segment.vector.BaseLongVectorValueSelector; import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; import org.apache.druid.segment.vector.NoFilterVectorOffset; @@ -44,7 +46,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; -public class DoubleFirstVectorAggregationTest extends InitializedNullHandlingTest +public class DoubleFirstVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final double[] VALUES = new double[]{7.8d, 11, 23.67, 60}; @@ -70,7 +72,8 @@ public class DoubleFirstVectorAggregationTest extends InitializedNullHandlingTes private DoubleFirstAggregatorFactory doubleFirstAggregatorFactory; private VectorColumnSelectorFactory selectorFactory; - private VectorValueSelector nonLongValueSelector; + private VectorValueSelector longValueSelector; + private VectorValueSelector doubleValueSelector; @Before public void setup() @@ -116,7 +119,7 @@ public int getCurrentVectorSize() } }; - nonLongValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + longValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( LONG_VALUES.length, 0, LONG_VALUES.length @@ -160,6 +163,22 @@ public int getCurrentVectorSize() } }; + doubleValueSelector = new BaseDoubleVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length)) + { + @Override + public double[] getDoubleVector() + { + return VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + }; + selectorFactory = new VectorColumnSelectorFactory() { @Override @@ -186,7 +205,9 @@ public VectorValueSelector makeValueSelector(String column) if (TIME_COL.equals(column)) { return timeSelector; } else if (FIELD_NAME_LONG.equals(column)) { - return nonLongValueSelector; + return longValueSelector; + } else if (FIELD_NAME.equals(column)) { + return doubleValueSelector; } return null; } @@ -231,15 +252,15 @@ public void testFactory() } @Test - public void initValueShouldInitZero() + public void testInit() { target.init(buf, 0); - Assert.assertEquals(Long.MAX_VALUE, buf.getLong(0)); + Assert.assertEquals(DateTimes.MAX.getMillis(), buf.getLong(0)); Assert.assertEquals(0, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @Test - public void aggregate() + public void testAggregate() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -248,7 +269,7 @@ public void aggregate() } @Test - public void aggregateWithNulls() + public void testAggregateWithNulls() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -257,7 +278,7 @@ public void aggregateWithNulls() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -271,7 +292,7 @@ public void aggregateBatchWithoutRows() } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 0, 2}; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregatorTest.java similarity index 96% rename from processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregatorTest.java index dc92164757c4..660fdc91e856 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/FloatFirstVectorAggregatorTest.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.VectorAggregator; @@ -45,7 +46,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; -public class FloatFirstVectorAggregationTest extends InitializedNullHandlingTest +public class FloatFirstVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final float[] VALUES = new float[]{7.2f, 15.6f, 2.1f, 150.0f}; @@ -262,15 +263,15 @@ public void testFactory() } @Test - public void initValueShouldBeZero() + public void testInit() { target.init(buf, 0); - Assert.assertEquals(Long.MAX_VALUE, buf.getLong(0)); + Assert.assertEquals(DateTimes.MAX.getMillis(), buf.getLong(0)); Assert.assertEquals(0.0f, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @Test - public void aggregate() + public void testAggregate() { target.init(buf, 0); target.aggregate(buf, 0, 0, VALUES.length); @@ -280,7 +281,7 @@ public void aggregate() } @Test - public void aggregateNulls1() + public void testAggregateNulls1() { target1.init(buf, 0); target1.aggregate(buf, 0, 0, VALUES.length); @@ -290,7 +291,7 @@ public void aggregateNulls1() } @Test - public void aggregateWithNulls() + public void testAggregateWithNulls() { target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); @@ -299,7 +300,7 @@ public void aggregateWithNulls() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -317,7 +318,7 @@ public void aggregateBatchWithoutRows() } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregatorTest.java similarity index 95% rename from processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java rename to processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregatorTest.java index 0ac8ff9275aa..24dd37dd0c24 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregationTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/LongFirstVectorAggregatorTest.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.firstlast.first; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; @@ -46,7 +47,7 @@ import java.util.concurrent.ThreadLocalRandom; -public class LongFirstVectorAggregationTest extends InitializedNullHandlingTest +public class LongFirstVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; private static final long[] VALUES = new long[]{7, 15, 2, 150}; @@ -189,7 +190,7 @@ public VectorValueSelector makeValueSelector(String column) { if (TIME_COL.equals(column)) { return timeSelector; - } else if (FIELD_NAME_LONG.equals(column)) { + } else if (FIELD_NAME_LONG.equals(column) || FIELD_NAME.equals(column)) { return nonLongValueSelector; } return null; @@ -235,15 +236,15 @@ public void testFactory() } @Test - public void initValueShouldInitZero() + public void testInit() { target.init(buf, 0); - Assert.assertEquals(Long.MAX_VALUE, buf.getLong(0)); + Assert.assertEquals(DateTimes.MAX.getMillis(), buf.getLong(0)); Assert.assertEquals(0, buf.getLong(FirstLastVectorAggregator.VALUE_OFFSET)); } @Test - public void aggregate() + public void testAggregate() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -252,7 +253,7 @@ public void aggregate() } @Test - public void aggregateWithNulls() + public void testAggregateWithNulls() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -261,7 +262,7 @@ public void aggregateWithNulls() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -279,7 +280,7 @@ public void aggregateBatchWithoutRows() } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java index 516c28f015f6..2652431e083c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/first/StringFirstVectorAggregatorTest.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation.firstlast.first; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongString; @@ -295,7 +296,7 @@ public void testFactory() } @Test - public void initValueShouldBeMaxDate() + public void testInit() { target.init(buf, 0); long initVal = buf.getLong(0); @@ -303,7 +304,7 @@ public void initValueShouldBeMaxDate() } @Test - public void aggregate() + public void testAggregate() { target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); @@ -325,7 +326,7 @@ public void testStringEarliestOnNonStringColumns() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -334,12 +335,12 @@ public void aggregateBatchWithoutRows() for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); Assert.assertEquals(times[i], result.lhs.longValue()); - Assert.assertEquals(VALUES[i], result.rhs); + Assert.assertEquals(NullHandling.nullToEmptyIfNeeded(VALUES[i]), result.rhs); } } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; @@ -349,7 +350,7 @@ public void aggregateBatchWithRows() for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); Assert.assertEquals(times[rows[i]], result.lhs.longValue()); - Assert.assertEquals(VALUES[rows[i]], result.rhs); + Assert.assertEquals(NullHandling.nullToEmptyIfNeeded(VALUES[rows[i]]), result.rhs); } } diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java index 1e91e16ce269..16e1bdb43892 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/DoubleLastVectorAggregatorTest.java @@ -19,13 +19,16 @@ package org.apache.druid.query.aggregation.firstlast.last; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongDouble; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.vector.BaseDoubleVectorValueSelector; import org.apache.druid.segment.vector.BaseLongVectorValueSelector; import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; import org.apache.druid.segment.vector.NoFilterVectorOffset; @@ -70,7 +73,8 @@ public class DoubleLastVectorAggregatorTest extends InitializedNullHandlingTest private DoubleLastAggregatorFactory doubleLastAggregatorFactory; private VectorColumnSelectorFactory selectorFactory; - private VectorValueSelector nonLongValueSelector; + private VectorValueSelector longValueSelector; + private VectorValueSelector doubleValueSelector; @Before public void setup() @@ -116,7 +120,7 @@ public int getCurrentVectorSize() } }; - nonLongValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( + longValueSelector = new BaseLongVectorValueSelector(new NoFilterVectorOffset( LONG_VALUES.length, 0, LONG_VALUES.length @@ -160,6 +164,22 @@ public int getCurrentVectorSize() } }; + doubleValueSelector = new BaseDoubleVectorValueSelector(new NoFilterVectorOffset(VALUES.length, 0, VALUES.length)) + { + @Override + public double[] getDoubleVector() + { + return VALUES; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return null; + } + }; + selectorFactory = new VectorColumnSelectorFactory() { @Override @@ -186,7 +206,9 @@ public VectorValueSelector makeValueSelector(String column) if (TIME_COL.equals(column)) { return timeSelector; } else if (FIELD_NAME_LONG.equals(column)) { - return nonLongValueSelector; + return longValueSelector; + } else if (FIELD_NAME.equals(column)) { + return doubleValueSelector; } return null; } @@ -231,15 +253,15 @@ public void testFactory() } @Test - public void initValueShouldInitZero() + public void testInit() { target.init(buf, 0); - double initVal = buf.getDouble(0); - Assert.assertEquals(0, initVal, EPSILON); + Assert.assertEquals(DateTimes.MIN.getMillis(), buf.getLong(0)); + Assert.assertEquals(0, buf.getDouble(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @Test - public void aggregate() + public void testAggregate() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -248,7 +270,7 @@ public void aggregate() } @Test - public void aggregateWithNulls() + public void testAggregateWithNulls() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -257,7 +279,7 @@ public void aggregateWithNulls() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -271,7 +293,7 @@ public void aggregateBatchWithoutRows() } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java index 589ac297f173..b19767a4ae9c 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/FloatLastVectorAggregatorTest.java @@ -20,9 +20,11 @@ package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongFloat; import org.apache.druid.query.aggregation.VectorAggregator; +import org.apache.druid.query.aggregation.firstlast.FirstLastVectorAggregator; import org.apache.druid.query.aggregation.firstlast.FloatFirstLastVectorAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.segment.column.ColumnCapabilities; @@ -232,19 +234,19 @@ public void testFactory() Assert.assertTrue(floatLastAggregatorFactory.canVectorize(selectorFactory)); VectorAggregator vectorAggregator = floatLastAggregatorFactory.factorizeVector(selectorFactory); Assert.assertNotNull(vectorAggregator); - Assert.assertEquals(FloatFirstLastVectorAggregator.class, vectorAggregator.getClass()); + Assert.assertEquals(FloatLastVectorAggregator.class, vectorAggregator.getClass()); } @Test - public void initValueShouldBeZero() + public void testInit() { target.init(buf, 0); - float initVal = buf.getFloat(0); - Assert.assertEquals(0.0f, initVal, EPSILON); + Assert.assertEquals(DateTimes.MIN.getMillis(), buf.getLong(0)); + Assert.assertEquals(0.0f, buf.getFloat(FirstLastVectorAggregator.VALUE_OFFSET), EPSILON); } @Test - public void aggregate() + public void testAggregate() { target.init(buf, 0); target.aggregate(buf, 0, 0, VALUES.length); @@ -254,7 +256,7 @@ public void aggregate() } @Test - public void aggregateWithNulls() + public void testAggregateWithNulls() { target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); @@ -263,7 +265,7 @@ public void aggregateWithNulls() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -281,7 +283,7 @@ public void aggregateBatchWithoutRows() } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java index 6eaa896558fc..04fffcd7d94d 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/LongLastVectorAggregatorTest.java @@ -20,6 +20,7 @@ package org.apache.druid.query.aggregation.firstlast.last; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongLong; import org.apache.druid.query.aggregation.VectorAggregator; @@ -45,11 +46,6 @@ import java.nio.ByteBuffer; import java.util.concurrent.ThreadLocalRandom; -// Check with empty arrays, nulls -// Check with folded up objects, and non-folded up objects -// Check with coercing objects -// Check both the functions that they work properly -// Check with pre-existing value on the buffer, and no pre-existing value on the buffer public class LongLastVectorAggregatorTest extends InitializedNullHandlingTest { private static final double EPSILON = 1e-5; @@ -193,7 +189,7 @@ public VectorValueSelector makeValueSelector(String column) { if (TIME_COL.equals(column)) { return timeSelector; - } else if (FIELD_NAME_LONG.equals(column)) { + } else if (FIELD_NAME_LONG.equals(column) || FIELD_NAME.equals(column)) { return nonLongValueSelector; } return null; @@ -239,16 +235,15 @@ public void testFactory() } @Test - public void initTest() + public void testInit() { target.init(buf, 0); - long initVal = buf.getLong(0); - Assert.assertEquals(Long.MIN_VALUE, initVal); + Assert.assertEquals(DateTimes.MIN.getMillis(), buf.getLong(0)); Assert.assertEquals(0, buf.getLong(FirstLastVectorAggregator.VALUE_OFFSET)); } @Test - public void aggregate() + public void testAggregate() { target.init(buf, 0); target.aggregate(buf, 0, 0, pairs.length); @@ -258,7 +253,7 @@ public void aggregate() } @Test - public void aggregateWithNulls() + public void testAggregateWithNulls() { target.aggregate(buf, 0, 0, pairs.length); Pair result = (Pair) target.get(buf, 0); @@ -267,7 +262,7 @@ public void aggregateWithNulls() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -285,7 +280,7 @@ public void aggregateBatchWithoutRows() } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; @@ -303,6 +298,67 @@ public void aggregateBatchWithRows() } } + @Test + public void testAggregateWithStringifiedLongs() + { + VectorObjectSelector objectSelector = new VectorObjectSelector() + { + @Override + public Object[] getObjectVector() + { + return new Object[]{"1000", "2000", "3000"}; + } + + @Override + public int getMaxVectorSize() + { + return 3; + } + + @Override + public int getCurrentVectorSize() + { + return 3; + } + }; + LongLastVectorAggregator ll = new LongLastVectorAggregator(createLongSelector(new Long[]{1L, 2L, 3L}), objectSelector); + clearBufferForPositions(0, 0); + ll.aggregate(buf, 0, 0, 3); + Pair val = (Pair) ll.get(buf, 0); + Assert.assertEquals(3, (long) val.lhs); + Assert.assertEquals(3000, (long) val.rhs); + } + + private VectorValueSelector createLongSelector(Long[] times) + { + return new BaseLongVectorValueSelector(new NoFilterVectorOffset(times.length, 0, times.length)) + { + @Override + public long[] getLongVector() + { + long[] timesCasted = new long[times.length]; + for (int i = 0; i < times.length; ++i) { + timesCasted[i] = times[i] == null ? 0 : times[i]; + } + return timesCasted; + } + + @Nullable + @Override + public boolean[] getNullVector() + { + if (NullHandling.replaceWithDefault()) { + return null; + } + boolean[] nulls = new boolean[times.length]; + for (int i = 0; i < times.length; ++i) { + nulls[i] = times[i] == null; + } + return nulls; + } + }; + } + private void clearBufferForPositions(int offset, int... positions) { for (int position : positions) { diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java index 861ab3d7487e..68aea948d6b1 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/firstlast/last/StringLastVectorAggregatorTest.java @@ -19,6 +19,8 @@ package org.apache.druid.query.aggregation.firstlast.last; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.aggregation.SerializablePairLongString; import org.apache.druid.query.aggregation.VectorAggregator; @@ -357,15 +359,15 @@ public void testStringLastOnNonStringColumns() } @Test - public void initValueShouldBeMinLong() + public void testInit() { target.init(buf, 0); long initVal = buf.getLong(0); - Assert.assertEquals(Long.MIN_VALUE, initVal); + Assert.assertEquals(DateTimes.MIN.getMillis(), initVal); } @Test - public void aggregate() + public void testAggregate() { target.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) target.get(buf, 0); @@ -374,7 +376,7 @@ public void aggregate() } @Test - public void aggregateNoOp() + public void testAggregateNoOp() { // Test that aggregates run just fine when the input field does not exist StringLastVectorAggregator aggregator = new StringLastVectorAggregator(null, selector, 10); @@ -382,7 +384,7 @@ public void aggregateNoOp() } @Test - public void aggregateBatchNoOp() + public void testAggregateBatchNoOp() { // Test that aggregates run just fine when the input field does not exist StringLastVectorAggregator aggregator = new StringLastVectorAggregator(null, selector, 10); @@ -393,7 +395,7 @@ public void aggregateBatchNoOp() } @Test - public void aggregateBatchWithoutRows() + public void testAggregateBatchWithoutRows() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -402,12 +404,12 @@ public void aggregateBatchWithoutRows() for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); Assert.assertEquals(times[i], result.lhs.longValue()); - Assert.assertEquals(VALUES[i], result.rhs); + Assert.assertEquals(NullHandling.nullToEmptyIfNeeded(VALUES[i]), result.rhs); } } @Test - public void aggregateBatchWithRows() + public void testAggregateBatchWithRows() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; @@ -417,12 +419,12 @@ public void aggregateBatchWithRows() for (int i = 0; i < positions.length; i++) { Pair result = (Pair) target.get(buf, positions[i] + positionOffset); Assert.assertEquals(times[rows[i]], result.lhs.longValue()); - Assert.assertEquals(VALUES[rows[i]], result.rhs); + Assert.assertEquals(NullHandling.nullToEmptyIfNeeded(VALUES[rows[i]]), result.rhs); } } @Test - public void aggregateSingleDim() + public void testAggregateSingleDim() { targetSingleDim.aggregate(buf, 0, 0, VALUES.length); Pair result = (Pair) targetSingleDim.get(buf, 0); @@ -431,7 +433,7 @@ public void aggregateSingleDim() } @Test - public void aggregateBatchWithoutRowsSingleDim() + public void testAggregateBatchWithoutRowsSingleDim() { int[] positions = new int[]{0, 43, 70}; int positionOffset = 2; @@ -445,7 +447,7 @@ public void aggregateBatchWithoutRowsSingleDim() } @Test - public void aggregateBatchWithRowsSingleDim() + public void testAggregateBatchWithRowsSingleDim() { int[] positions = new int[]{0, 43, 70}; int[] rows = new int[]{3, 2, 0}; From 27abb363b5118b67070962d2a3103536e77e9fd2 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 26 Apr 2024 11:31:49 +0530 Subject: [PATCH 08/10] fixup --- .../query/aggregation/firstlast/FirstLastUtils.java | 9 +++------ .../aggregation/firstlast/FirstLastVectorAggregator.java | 9 ++------- .../firstlast/first/StringFirstAggregatorFactory.java | 9 +++++---- .../firstlast/last/StringLastAggregatorFactory.java | 1 - 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java index 460118be9278..1dce1472126d 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastUtils.java @@ -78,8 +78,7 @@ public static SerializablePairLongDouble readDoublePairFromVectorSelectors( if (pair.lhs == null) { return null; } - time = pair.lhs; - value = pair.rhs; + return pair; } else { if (timeNullityVector != null && timeNullityVector[index]) { // Donot aggregate pairs where time is unknown @@ -111,8 +110,7 @@ public static SerializablePairLongFloat readFloatPairFromVectorSelectors( if (pair.lhs == null) { return null; } - time = pair.lhs; - value = pair.rhs; + return pair; } else { if (timeNullityVector != null && timeNullityVector[index]) { // Donot aggregate pairs where time is unknown @@ -143,8 +141,7 @@ public static SerializablePairLongLong readLongPairFromVectorSelectors( if (pair.lhs == null) { return null; } - time = pair.lhs; - value = pair.rhs; + return pair; } else { if (timeNullityVector != null && timeNullityVector[index]) { // Donot aggregate pairs where time is unknown diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java index 8c21ab77275b..129465a6d591 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/firstlast/FirstLastVectorAggregator.java @@ -49,12 +49,7 @@ public abstract class FirstLastVectorAggregator Date: Fri, 26 Apr 2024 11:52:33 +0530 Subject: [PATCH 09/10] fixup --- .../org/apache/druid/segment/column/SchemaPayloadPlusTest.java | 2 +- .../java/org/apache/druid/segment/column/SchemaPayloadTest.java | 2 +- .../apache/druid/segment/column/SegmentSchemaMappingTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadPlusTest.java b/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadPlusTest.java index ac501c1524e8..9fb08fb74ff9 100644 --- a/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadPlusTest.java +++ b/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadPlusTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.SchemaPayloadPlus; import org.apache.druid.segment.TestHelper; diff --git a/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadTest.java b/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadTest.java index 0878e92d2d8c..dfb352775638 100644 --- a/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadTest.java +++ b/processing/src/test/java/org/apache/druid/segment/column/SchemaPayloadTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.TestHelper; import org.junit.Assert; diff --git a/processing/src/test/java/org/apache/druid/segment/column/SegmentSchemaMappingTest.java b/processing/src/test/java/org/apache/druid/segment/column/SegmentSchemaMappingTest.java index 38c9362a11ce..4272e67433cc 100644 --- a/processing/src/test/java/org/apache/druid/segment/column/SegmentSchemaMappingTest.java +++ b/processing/src/test/java/org/apache/druid/segment/column/SegmentSchemaMappingTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.query.aggregation.last.StringLastAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.last.StringLastAggregatorFactory; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.SchemaPayloadPlus; import org.apache.druid.segment.SegmentMetadata; From 670a3501798e79f834ff48e9734b2d92c6d2ec18 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 26 Apr 2024 12:03:15 +0530 Subject: [PATCH 10/10] fixup --- .../apache/druid/segment/metadata/FingerprintGeneratorTest.java | 2 +- .../druid/segment/metadata/SegmentSchemaBackFillQueueTest.java | 2 +- .../apache/druid/segment/metadata/SegmentSchemaManagerTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/FingerprintGeneratorTest.java b/server/src/test/java/org/apache/druid/segment/metadata/FingerprintGeneratorTest.java index 62ef3783699c..a1a49d91bbb6 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/FingerprintGeneratorTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/FingerprintGeneratorTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaBackFillQueueTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaBackFillQueueTest.java index e014c21e961b..238b5acc5442 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaBackFillQueueTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaBackFillQueueTest.java @@ -28,7 +28,7 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutors; import org.apache.druid.metadata.TestDerbyConnector; import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaManagerTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaManagerTest.java index a79b89f79b39..ed6a567644d4 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaManagerTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentSchemaManagerTest.java @@ -28,7 +28,7 @@ import org.apache.druid.metadata.MetadataStorageTablesConfig; import org.apache.druid.metadata.TestDerbyConnector; import org.apache.druid.query.aggregation.AggregatorFactory; -import org.apache.druid.query.aggregation.first.LongFirstAggregatorFactory; +import org.apache.druid.query.aggregation.firstlast.first.LongFirstAggregatorFactory; import org.apache.druid.segment.SchemaPayload; import org.apache.druid.segment.SchemaPayloadPlus; import org.apache.druid.segment.TestHelper;