diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java index 60c5b29f46c9..a5a7c4ca3905 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java @@ -82,6 +82,7 @@ import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; +import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.SqlVectorizedExpressionResultConsistencyTest; import org.apache.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator; import org.apache.druid.sql.calcite.aggregation.SqlAggregationModule; @@ -343,8 +344,12 @@ public void setup() throws JsonProcessingException try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, getQuery(), getContext())) { final PlannerResult plannerResult = planner.plan(); final Sequence resultSequence = plannerResult.run().getResults(); - final int rowCount = resultSequence.toList().size(); + final List results = resultSequence.toList(); + final int rowCount = results.size(); log.info("Total result row count:" + rowCount); + if (rowCount < 10) { + log.info(BaseCalciteQueryTest.resultsToString("query results", results)); + } } catch (Throwable ex) { log.warn(ex, "failed to count rows"); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 0fc2a8236918..cc9e7b888dad 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -156,8 +156,10 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark // numeric no lhs null "SELECT NVL(long1, long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2", "SELECT NVL(long1, long5 + long3), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2", - "SELECT CASE WHEN MOD(long1, 2) = 0 THEN -1 WHEN MOD(long1, 2) = 1 THEN long2 / MOD(long1, 2) ELSE long3 END FROM expressions GROUP BY 1" - + "SELECT CASE WHEN MOD(long1, 2) = 0 THEN -1 WHEN MOD(long1, 2) = 1 THEN long2 / MOD(long1, 2) ELSE long3 END FROM expressions GROUP BY 1", + // cast + "SELECT CAST(string1 as BIGINT) + CAST(string3 as DOUBLE) + long3, COUNT(*) FROM expressions GROUP BY 1 ORDER BY 2", + "SELECT COUNT(*), SUM(CAST(string1 as BIGINT) + CAST(string3 as BIGINT)) FROM expressions WHERE double3 < 1010.0 AND double3 > 100.0" ); @Param({ @@ -229,7 +231,9 @@ public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark "55", "56", "57", - "58" + "58", + "59", + "60" }) private String query; diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalNumericBindingVector.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalNumericBindingVector.java new file mode 100644 index 000000000000..c65fede9cb8e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalNumericBindingVector.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.error.DruidException; +import org.apache.druid.math.expr.Evals; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.ExpressionType; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +import javax.annotation.Nullable; +import java.util.function.Supplier; + +/** + * {@link ExprEvalVector} backed directly by an underlying {@link Expr.VectorInputBinding} for numeric type bindings + */ +public class ExprEvalNumericBindingVector implements ExprEvalVector +{ + private final ExpressionType expressionType; + private final Expr.VectorInputBinding bindings; + private final String bindingName; + private final Supplier vectorCacheSupplier; + + @MonotonicNonNull + private Object[] objects; + + public ExprEvalNumericBindingVector( + ExpressionType expressionType, + Expr.VectorInputBinding bindings, + String name, + Supplier vectorCacheSupplier + ) + { + this.expressionType = expressionType; + this.bindings = bindings; + this.bindingName = name; + this.vectorCacheSupplier = vectorCacheSupplier; + } + + @Override + public ExpressionType getType() + { + return expressionType; + } + + @Override + public T values() + { + if (expressionType.is(ExprType.LONG)) { + return (T) getLongVector(); + } else if (expressionType.is(ExprType.DOUBLE)) { + return (T) getDoubleVector(); + } + throw DruidException.defensive( + "Non-numeric type[%s] for binding[%s] used with numeric binding vector", + expressionType, + bindingName + ); + } + + @Override + public long[] getLongVector() + { + return bindings.getLongVector(bindingName); + } + + @Override + public double[] getDoubleVector() + { + return bindings.getDoubleVector(bindingName); + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return bindings.getNullVector(bindingName); + } + + @Override + public Object[] getObjectVector() + { + if (objects == null) { + objects = vectorCacheSupplier.get().objects; + if (expressionType.is(ExprType.LONG)) { + final long[] longs = bindings.getLongVector(bindingName); + final boolean[] numericNulls = bindings.getNullVector(bindingName); + if (numericNulls != null) { + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { + objects[i] = numericNulls[i] ? null : longs[i]; + } + } else { + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { + objects[i] = longs[i]; + } + } + } else if (expressionType.is(ExprType.DOUBLE)) { + objects = vectorCacheSupplier.get().objects; + final double[] doubles = bindings.getDoubleVector(bindingName); + final boolean[] numericNulls = bindings.getNullVector(bindingName); + if (numericNulls != null) { + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { + objects[i] = numericNulls[i] ? null : doubles[i]; + } + } else { + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { + objects[i] = doubles[i]; + } + } + } else { + throw DruidException.defensive( + "Non-numeric type[%s] for binding[%s] used with numeric binding vector", + expressionType, + bindingName + ); + } + } + return objects; + } + + @Override + public boolean elementAsBoolean(int index) + { + if (expressionType.is(ExprType.LONG)) { + final long[] longs = bindings.getLongVector(bindingName); + final boolean[] numericNulls = bindings.getNullVector(bindingName); + if (numericNulls != null && numericNulls[index]) { + return Evals.asBoolean(0L); + } + return Evals.asBoolean(longs[index]); + } else if (expressionType.is(ExprType.DOUBLE)) { + final double[] doubles = bindings.getDoubleVector(bindingName); + final boolean[] numericNulls = bindings.getNullVector(bindingName); + if (numericNulls != null && numericNulls[index]) { + return Evals.asBoolean(0.0); + } + return Evals.asBoolean(doubles[index]); + } + throw DruidException.defensive( + "Non-numeric type[%s] for binding[%s] used with numeric binding vector", + expressionType, + bindingName + ); + } + + public static VectorCache createCache(int maxSize) + { + return new VectorCache(maxSize); + } + + /** + * Container to re-use a allocated objects across various {@link ExprEvalNumericBindingVector} instances + */ + public static class VectorCache + { + private final Object[] objects; + + private VectorCache(int maxSize) + { + this.objects = new Object[maxSize]; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalBindingVector.java b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectBindingVector.java similarity index 50% rename from processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalBindingVector.java rename to processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectBindingVector.java index 72750c977e06..8b8bc5d6dfcc 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalBindingVector.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectBindingVector.java @@ -26,15 +26,17 @@ import org.apache.druid.math.expr.ExpressionType; import javax.annotation.Nullable; +import java.util.function.Supplier; /** - * {@link ExprEvalVector} backed directly by an underlying {@link Expr.VectorInputBinding} + * {@link ExprEvalVector} backed directly by an underlying {@link Expr.VectorInputBinding} for object type bindings */ -public class ExprEvalBindingVector implements ExprEvalVector +public class ExprEvalObjectBindingVector implements ExprEvalVector { private final ExpressionType expressionType; private final Expr.VectorInputBinding bindings; private final String bindingName; + private final Supplier vectorCacheSupplier; @Nullable private long[] longs; @@ -43,17 +45,18 @@ public class ExprEvalBindingVector implements ExprEvalVector @Nullable private boolean[] numericNulls; - private Object[] objects; - public ExprEvalBindingVector( + public ExprEvalObjectBindingVector( ExpressionType expressionType, Expr.VectorInputBinding bindings, - String name + String name, + Supplier vectorCacheSupplier ) { this.expressionType = expressionType; this.bindings = bindings; this.bindingName = name; + this.vectorCacheSupplier = vectorCacheSupplier; } @Override @@ -65,33 +68,20 @@ public ExpressionType getType() @Override public T values() { - if (expressionType.is(ExprType.LONG)) { - return (T) getLongVector(); - } else if (expressionType.is(ExprType.DOUBLE)) { - return (T) getDoubleVector(); - } - return (T) bindings.getObjectVector(bindingName); + return (T) getObjectVector(); } @Override public long[] getLongVector() { - if (expressionType.isNumeric()) { - longs = bindings.getLongVector(bindingName); - } else { - computeNumbers(); - } + computeNumbers(); return longs; } @Override public double[] getDoubleVector() { - if (expressionType.isNumeric()) { - doubles = bindings.getDoubleVector(bindingName); - } else { - computeNumbers(); - } + computeNumbers(); return doubles; } @@ -99,90 +89,36 @@ public double[] getDoubleVector() @Override public boolean[] getNullVector() { - if (expressionType.isNumeric()) { - numericNulls = bindings.getNullVector(bindingName); - } else { - computeNumbers(); - } + computeNumbers(); return numericNulls; } @Override public Object[] getObjectVector() { - if (expressionType.is(ExprType.LONG)) { - final long[] values = bindings.getLongVector(bindingName); - final boolean[] nulls = bindings.getNullVector(bindingName); - objects = new Long[values.length]; - if (nulls != null) { - for (int i = 0; i < values.length; i++) { - objects[i] = nulls[i] ? null : values[i]; - } - } else { - for (int i = 0; i < values.length; i++) { - objects[i] = values[i]; - } - } - } else if (expressionType.is(ExprType.DOUBLE)) { - final double[] values = bindings.getDoubleVector(bindingName); - final boolean[] nulls = bindings.getNullVector(bindingName); - objects = new Double[values.length]; - if (nulls != null) { - for (int i = 0; i < values.length; i++) { - objects[i] = nulls[i] ? null : values[i]; - } - } else { - for (int i = 0; i < values.length; i++) { - objects[i] = values[i]; - } - } - } else { - objects = bindings.getObjectVector(bindingName); - } - return objects; + return bindings.getObjectVector(bindingName); } @Override public boolean elementAsBoolean(int index) { - if (expressionType.is(ExprType.LONG)) { - if (longs == null) { - // populate stuff - getLongVector(); - } - if (numericNulls != null && numericNulls[index]) { - return Evals.asBoolean(0L); - } - return Evals.asBoolean(longs[index]); - } else if (expressionType.is(ExprType.DOUBLE)) { - if (doubles == null) { - getDoubleVector(); - } - if (numericNulls != null && numericNulls[index]) { - return Evals.asBoolean(0.0); - } - return Evals.asBoolean(doubles[index]); + final Object[] objects = bindings.getObjectVector(bindingName); + if (expressionType.is(ExprType.STRING)) { + return Evals.asBoolean((String) objects[index]); } else { - if (objects == null) { - getObjectVector(); - } - if (expressionType.is(ExprType.STRING)) { - return Evals.asBoolean((String) objects[index]); - } else { - return ExprEval.ofType(expressionType, objects[index]).asBoolean(); - } + return ExprEval.ofType(expressionType, objects[index]).asBoolean(); } } private void computeNumbers() { - final Object[] values = getObjectVector(); + final Object[] values = bindings.getObjectVector(bindingName); if (longs == null) { - longs = new long[values.length]; - doubles = new double[values.length]; - numericNulls = new boolean[values.length]; + longs = vectorCacheSupplier.get().longs; + doubles = vectorCacheSupplier.get().doubles; + numericNulls = vectorCacheSupplier.get().nulls; boolean isString = expressionType.is(ExprType.STRING); - for (int i = 0; i < values.length; i++) { + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { if (isString) { Number n = ExprEval.computeNumber(Evals.asString(values[i])); if (n != null) { @@ -204,4 +140,26 @@ private void computeNumbers() } } } + + public static VectorCache createCache(int maxSize) + { + return new VectorCache(maxSize); + } + + /** + * Container to re-use a allocated objects across various {@link ExprEvalObjectBindingVector} instances + */ + public static class VectorCache + { + private final long[] longs; + private final double[] doubles; + private final boolean[] nulls; + + private VectorCache(int maxSize) + { + this.longs = new long[maxSize]; + this.doubles = new double[maxSize]; + this.nulls = new boolean[maxSize]; + } + } } diff --git a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index 121c907fc3b3..883254fe4dea 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/processing/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -872,6 +872,10 @@ static final class IdentifierVectorProcessor implements ExprVectorProcessor evalVector(Expr.VectorInputBinding bindings) { - return new ExprEvalBindingVector<>(outputType, bindings, bindingName); + if (outputType.isNumeric()) { + return new ExprEvalNumericBindingVector<>(outputType, bindings, bindingName, this::getNumericCache); + } + return new ExprEvalObjectBindingVector<>(outputType, bindings, bindingName, this::getObjectCache); } @Override @@ -897,5 +904,21 @@ public int maxVectorSize() { return maxVectorSize; } + + private ExprEvalObjectBindingVector.VectorCache getObjectCache() + { + if (objectCache == null) { + objectCache = ExprEvalObjectBindingVector.createCache(maxVectorSize); + } + return objectCache; + } + + private ExprEvalNumericBindingVector.VectorCache getNumericCache() + { + if (numericCache == null) { + numericCache = ExprEvalNumericBindingVector.createCache(maxVectorSize); + } + return numericCache; + } } }