From 88f898fbf4276878b341b063fa525225c589b6bb Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 16 Sep 2020 04:10:20 -0700 Subject: [PATCH 01/17] vectorized expression virtual columns --- benchmarks/pom.xml | 5 + .../ExpressionVectorSelectorBenchmark.java | 302 ++++++++++ .../query/SqlExpressionBenchmark.java | 361 ++++++++++++ .../apache/druid/math/expr/ApplyFunction.java | 12 + .../math/expr/BinaryLogicalOperatorExpr.java | 466 +++++++++++++++ .../math/expr/BinaryMathOperatorExpr.java | 503 +++++++++++++++- .../apache/druid/math/expr/ConstantExpr.java | 305 +++++++--- .../java/org/apache/druid/math/expr/Expr.java | 75 +++ .../org/apache/druid/math/expr/ExprEval.java | 29 +- .../druid/math/expr/ExprListenerImpl.java | 4 +- .../org/apache/druid/math/expr/ExprType.java | 22 + .../org/apache/druid/math/expr/Exprs.java | 16 + .../org/apache/druid/math/expr/Function.java | 557 ++++++++++++++++++ .../druid/math/expr/FunctionalExpr.java | 39 ++ .../druid/math/expr/IdentifierExpr.java | 68 ++- .../druid/math/expr/UnaryOperatorExpr.java | 80 ++- .../BivariateFunctionVectorProcessor.java | 85 +++ .../vector/CastToDoubleVectorProcessor.java | 44 ++ .../vector/CastToLongVectorProcessor.java | 44 ++ .../vector/CastToStringVectorProcessor.java | 44 ++ .../vector/CastToTypeVectorProcessor.java | 56 ++ ...eLongBivariateFunctionVectorProcessor.java | 60 ++ ...oubleBivariateFunctionVectorProcessor.java | 60 ++ ...ubleUnivariateFunctionVectorProcessor.java | 56 ++ .../expr/vector/DoubleVectorExprEval.java | 68 +++ ...ublesBivariateFunctionVectorProcessor.java | 60 ++ ...ubleUnivariateFunctionVectorProcessor.java | 56 ++ ...LongUnivariateFunctionVectorProcessor.java | 56 ++ .../math/expr/vector/LongVectorExprEval.java | 63 ++ ...LongsBivariateFunctionVectorProcessor.java | 60 ++ ...oubleBivariateFunctionVectorProcessor.java | 60 ++ ...LongUnivariateFunctionVectorProcessor.java | 42 ++ .../expr/vector/StringVectorExprEval.java | 113 ++++ ...ivariateFunctionVectorObjectProcessor.java | 61 ++ .../UnivariateFunctionVectorProcessor.java | 77 +++ .../math/expr/vector/VectorExprEval.java | 73 +++ .../math/expr/vector/VectorExprProcessor.java | 30 + .../org/apache/druid/math/expr/ExprTest.java | 4 +- .../apache/druid/math/expr/FunctionTest.java | 4 +- .../druid/math/expr/VectorExprSanityTest.java | 423 +++++++++++++ .../query/aggregation/AggregatorUtil.java | 21 +- .../DoubleMaxAggregatorFactory.java | 18 - .../DoubleMinAggregatorFactory.java | 18 - .../DoubleSumAggregatorFactory.java | 20 - .../FloatMaxAggregatorFactory.java | 19 - .../FloatMinAggregatorFactory.java | 18 - .../FloatSumAggregatorFactory.java | 18 - .../aggregation/LongMaxAggregatorFactory.java | 18 - .../aggregation/LongMinAggregatorFactory.java | 18 - .../aggregation/LongSumAggregatorFactory.java | 18 - .../SimpleDoubleAggregatorFactory.java | 23 + .../SimpleFloatAggregatorFactory.java | 23 + .../SimpleLongAggregatorFactory.java | 23 + .../expression/TimestampFloorExprMacro.java | 28 + .../QueryableIndexColumnSelectorFactory.java | 5 +- .../apache/druid/segment/VirtualColumn.java | 6 + .../apache/druid/segment/VirtualColumns.java | 21 +- .../column/StringDictionaryEncodedColumn.java | 51 ++ .../generator/GeneratorBasicSchemas.java | 46 ++ ...IncrementalIndexColumnSelectorFactory.java | 2 +- ...yableIndexVectorColumnSelectorFactory.java | 5 +- .../segment/virtual/ExpressionSelectors.java | 197 ++++++- .../ExpressionVectorObjectSelector.java | 55 ++ .../ExpressionVectorValueSelector.java | 82 +++ .../virtual/ExpressionVirtualColumn.java | 83 +++ .../VirtualizedColumnSelectorFactory.java | 2 +- .../query/groupby/GroupByQueryRunnerTest.java | 48 +- .../timeseries/TimeseriesQueryRunnerTest.java | 3 - .../segment/generator/DataGeneratorTest.java | 68 +++ .../apache/druid/server/QueryStackTests.java | 10 +- .../druid/sql/calcite/CalciteQueryTest.java | 24 - 71 files changed, 5178 insertions(+), 356 deletions(-) create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java create mode 100644 core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index f00b93125c51..dc4868d4611e 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -47,6 +47,11 @@ ${jmh.version} provided + + org.easymock + easymock + 4.0.2 + org.apache.druid druid-processing diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java new file mode 100644 index 000000000000..8e83e6f5e83d --- /dev/null +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.benchmark; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Parser; +import org.apache.druid.query.expression.TestExprMacroTable; +import org.apache.druid.segment.ColumnInspector; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.Cursor; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.QueryableIndexStorageAdapter; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.generator.GeneratorBasicSchemas; +import org.apache.druid.segment.generator.GeneratorSchemaInfo; +import org.apache.druid.segment.generator.SegmentGenerator; +import org.apache.druid.segment.vector.VectorCursor; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionSelectors; +import org.apache.druid.segment.virtual.ExpressionVirtualColumn; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.junit.Assert; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + + +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 3) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public class ExpressionVectorSelectorBenchmark +{ + static { + NullHandling.initializeForTests(); + } + + @Param({"1000000"}) + private int rowsPerSegment; + + @Param({"false", "true"}) + private boolean vectorize; + + @Param({ + "long1 * long2", + "double1 * double3", + "float1 + float3", + "(long1 - long4) / double3", + "max(double3, double5)", + "min(double4, double1)", + "cos(float3)", + "sin(long4)", + "parse_long(string1)", + "parse_long(string1) * double3", + "parse_long(string5) * parse_long(string1)", + "parse_long(string5) * parse_long(string1) * double3" + }) + private String expression; + + private QueryableIndex index; + private Closer closer; + + @Nullable + private ExprType outputType; + + @Setup(Level.Trial) + public void setup() + { + this.closer = Closer.create(); + + final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); + + final DataSegment dataSegment = DataSegment.builder() + .dataSource("foo") + .interval(schemaInfo.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(); + + final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator()); + this.index = closer.register( + segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, rowsPerSegment) + ); + + Expr parsed = Parser.parse(expression, ExprMacroTable.nil()); + outputType = parsed.getOutputType( + ExpressionSelectors.makeInspectorBindingTypes(new ColumnInspector() + { + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + return QueryableIndexStorageAdapter.getColumnCapabilities(index, column); + } + }) + ); + checkSanity(); + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception + { + closer.close(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void scan(Blackhole blackhole) + { + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn( + "v", + expression, + ExprType.toValueType(outputType), + TestExprMacroTable.INSTANCE + ) + ) + ); + if (vectorize) { + VectorCursor cursor = new QueryableIndexStorageAdapter(index).makeVectorCursor( + null, + index.getDataInterval(), + virtualColumns, + false, + 512, + null + ); + if (outputType.isNumeric()) { + VectorValueSelector selector = cursor.getColumnSelectorFactory().makeValueSelector("v"); + if (outputType.equals(ExprType.DOUBLE)) { + while (!cursor.isDone()) { + blackhole.consume(selector.getDoubleVector()); + blackhole.consume(selector.getNullVector()); + cursor.advance(); + } + } else { + while (!cursor.isDone()) { + blackhole.consume(selector.getLongVector()); + blackhole.consume(selector.getNullVector()); + cursor.advance(); + } + } + closer.register(cursor); + } + } else { + Sequence cursors = new QueryableIndexStorageAdapter(index).makeCursors( + null, + index.getDataInterval(), + virtualColumns, + Granularities.ALL, + false, + null + ); + + int rowCount = cursors + .map(cursor -> { + final ColumnValueSelector selector = cursor.getColumnSelectorFactory().makeColumnValueSelector("v"); + int rows = 0; + while (!cursor.isDone()) { + blackhole.consume(selector.getObject()); + rows++; + cursor.advance(); + } + return rows; + }).accumulate(0, (acc, in) -> acc + in); + + blackhole.consume(rowCount); + } + } + + private void checkSanity() + { + final List results = new ArrayList<>(rowsPerSegment); + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn( + "v", + expression, + ExprType.toValueType(outputType), + TestExprMacroTable.INSTANCE + ) + ) + ); + VectorCursor cursor = new QueryableIndexStorageAdapter(index).makeVectorCursor( + null, + index.getDataInterval(), + virtualColumns, + false, + 512, + null + ); + + VectorValueSelector selector = null; + VectorObjectSelector objectSelector = null; + if (outputType.isNumeric()) { + selector = cursor.getColumnSelectorFactory().makeValueSelector("v"); + } else { + objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v"); + } + int rowCount = 0; + while (!cursor.isDone()) { + boolean[] nulls; + switch (outputType) { + case LONG: + nulls = selector.getNullVector(); + long[] longs = selector.getLongVector(); + for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) { + results.add(nulls != null && nulls[i] ? null : longs[i]); + } + break; + case DOUBLE: + nulls = selector.getNullVector(); + double[] doubles = selector.getDoubleVector(); + for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) { + results.add(nulls != null && nulls[i] ? null : doubles[i]); + } + break; + case STRING: + Object[] objects = objectSelector.getObjectVector(); + for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) { + results.add(objects[i]); + } + break; + } + + cursor.advance(); + } + closer.register(cursor); + + Sequence cursors = new QueryableIndexStorageAdapter(index).makeCursors( + null, + index.getDataInterval(), + virtualColumns, + Granularities.ALL, + false, + null + ); + + int rowCountCursor = cursors + .map(nonVectorized -> { + final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory().makeColumnValueSelector("v"); + int rows = 0; + while (!nonVectorized.isDone()) { + Assert.assertEquals(StringUtils.format("Failed at row %s", rows), nonSelector.getObject(), results.get(rows)); + rows++; + nonVectorized.advance(); + } + return rows; + }).accumulate(0, (acc, in) -> acc + in); + + Assert.assertTrue(rowCountCursor > 0); + } +} diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java new file mode 100644 index 000000000000..328e44680760 --- /dev/null +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.benchmark.query; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.DruidProcessingConfig; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.generator.GeneratorBasicSchemas; +import org.apache.druid.segment.generator.GeneratorSchemaInfo; +import org.apache.druid.segment.generator.SegmentGenerator; +import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.security.AuthTestUtils; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.DruidPlanner; +import org.apache.druid.sql.calcite.planner.PlannerConfig; +import org.apache.druid.sql.calcite.planner.PlannerFactory; +import org.apache.druid.sql.calcite.planner.PlannerResult; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.junit.Assert; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Benchmark that tests various SQL queries. + */ +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 3) +@Measurement(iterations = 5) +public class SqlExpressionBenchmark +{ + private static final Logger log = new Logger(SqlExpressionBenchmark.class); + + static { + NullHandling.initializeForTests(); + Calcites.setSystemProperties(); + } + static DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() + { + @Override + public int intermediateComputeSizeBytes() + { + return 512 * 1024 * 1024; + } + + @Override + public int getNumMergeBuffers() + { + return 3; + } + + @Override + public int getNumThreads() + { + return 1; + } + + @Override + public boolean useParallelMergePoolConfigured() + { + return true; + } + + @Override + public String getFormatString() + { + return "benchmarks-processing-%s"; + } + }; + + + private static final List QUERIES = ImmutableList.of( + // =========================== + // non-expression reference queries + // =========================== + // 0: non-expression timeseries reference, 1 columns + "SELECT SUM(long1) FROM foo", + // 1: non-expression timeseries reference, 2 columns + "SELECT SUM(long1), SUM(long2) FROM foo", + // 2: non-expression timeseries reference, 3 columns + "SELECT SUM(long1), SUM(long4), SUM(double1) FROM foo", + // 3: non-expression timeseries reference, 4 columns + "SELECT SUM(long1), SUM(long4), SUM(double1), SUM(float3) FROM foo", + // 4: non-expression timeseries reference, 5 columns + "SELECT SUM(long1), SUM(long4), SUM(double1), SUM(float3), SUM(long5) FROM foo", + // 5: group by non-expr with 1 agg + "SELECT string2, SUM(long1) FROM foo GROUP BY 1 ORDER BY 2", + // 6: group by non-expr with 2 agg + "SELECT string2, SUM(long1), SUM(double3) FROM foo GROUP BY 1 ORDER BY 2", + // =========================== + // expressions + // =========================== + // 7: math op - 2 longs + "SELECT SUM(long1 * long2) FROM foo", + // 8: mixed math - 2 longs, 1 double + "SELECT SUM((long1 * long2) / double1) FROM foo", + // 9: mixed math - 2 longs, 1 double, 1 float + "SELECT SUM(float3 + ((long1 * long4)/double1)) FROM foo", + // 10: mixed math - 3 longs, 1 double, 1 float + "SELECT SUM(long5 - (float3 + ((long1 * long4)/double1))) FROM foo", + // 11: all same math op - 3 longs, 1 double, 1 float + "SELECT SUM(long5 * float3 * long1 * long4 * double1) FROM foo", + // 12: cos + "SELECT cos(double2) FROM foo", + // 13: unary negate + "SELECT SUM(-long4) FROM foo", + // 14: string long + "SELECT SUM(PARSE_LONG(string1)) FROM foo", + // 15: string longer + "SELECT SUM(PARSE_LONG(string3)) FROM foo", + // 16: time floor, non-expr col + reg agg + "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(double4) FROM foo GROUP BY 1,2 ORDER BY 3", + // 17: time floor, non-expr col + expr agg + "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3", + // 18: time floor + non-expr agg (timeseries) (non-expression reference) + "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1) FROM foo GROUP BY 1 ORDER BY 1", + // 19: time floor + expr agg (timeseries) + "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 1", + // 20: time floor + non-expr agg (group by) + "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1) FROM foo GROUP BY 1 ORDER BY 2", + // 21: time floor + expr agg (group by) + "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2", + // 22: time floor offset by 1 day + non-expr agg (group by) + "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1) FROM foo GROUP BY 1 ORDER BY 1", + // 23: time floor offset by 1 day + expr agg (group by) + "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 1", + // 24: group by long expr with non-expr agg + "SELECT (long1 * long2), SUM(double1) FROM foo GROUP BY 1 ORDER BY 2", + // 25: group by non-expr with expr agg + "SELECT string2, SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2" + ); + + @Param({"5000000"}) + private int rowsPerSegment; + + @Param({"false", "force"}) + private String vectorize; + + @Param({ + // non-expression reference + "0", + "1", + "2", + "3", + "4", + "5", + "6", + // expressions + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25" + }) + private String query; + + @Nullable + private PlannerFactory plannerFactory; + private Closer closer = Closer.create(); + + @Setup(Level.Trial) + public void setup() throws Exception + { + final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); + + final DataSegment dataSegment = DataSegment.builder() + .dataSource("foo") + .interval(schemaInfo.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(); + + final PlannerConfig plannerConfig = new PlannerConfig(); + + final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator()); + log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].", segmentGenerator.getCacheDir(), rowsPerSegment); + final QueryableIndex index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment); + + final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate( + closer, + PROCESSING_CONFIG + ); + + final SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + dataSegment, + index + ); + closer.register(walker); + + final SchemaPlus rootSchema = + CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER); + plannerFactory = new PlannerFactory( + rootSchema, + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createOperatorTable(), + CalciteTests.createExprMacroTable(), + plannerConfig, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + CalciteTests.getJsonMapper(), + CalciteTests.DRUID_SCHEMA_NAME + ); + + checkSanity(); + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception + { + closer.close(); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void querySql(Blackhole blackhole) throws Exception + { + final Map context = ImmutableMap.of("vectorize", vectorize); + final AuthenticationResult authenticationResult = NoopEscalator.getInstance() + .createEscalatedAuthenticationResult(); + try (final DruidPlanner planner = plannerFactory.createPlanner(context, ImmutableList.of(), authenticationResult)) { + final PlannerResult plannerResult = planner.plan(QUERIES.get(Integer.parseInt(query))); + final Sequence resultSequence = plannerResult.run(); + final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in); + blackhole.consume(lastRow); + } + } + + public void checkSanity() throws Exception + { + final Map vector = ImmutableMap.of("vectorize", true); + final Map nonvector = ImmutableMap.of("vectorize", false); + final AuthenticationResult authenticationResult = NoopEscalator.getInstance() + .createEscalatedAuthenticationResult(); + + try ( + final DruidPlanner vectorPlanner = plannerFactory.createPlanner(vector, ImmutableList.of(), authenticationResult); + final DruidPlanner nonVectorPlanner = plannerFactory.createPlanner(nonvector, ImmutableList.of(), authenticationResult) + ) { + final PlannerResult vectorPlan = vectorPlanner.plan(QUERIES.get(Integer.parseInt(query))); + final PlannerResult nonVectorPlan = nonVectorPlanner.plan(QUERIES.get(Integer.parseInt(query))); + final Sequence vectorSequence = vectorPlan.run(); + final Sequence nonVectorSequence = nonVectorPlan.run(); + Yielder vectorizedYielder = Yielders.each(vectorSequence); + Yielder nonVectorizedYielder = Yielders.each(nonVectorSequence); + int row = 0; + int misMatch = 0; + while (!vectorizedYielder.isDone() && !nonVectorizedYielder.isDone()) { + Object[] vectorGet = vectorizedYielder.get(); + Object[] nonVectorizedGet = nonVectorizedYielder.get(); + + try { + if (vectorGet[0] instanceof Float || vectorGet[0] instanceof Double) { + Assert.assertArrayEquals( + StringUtils.format( + "Double results differed at row %s (%s : %s)", + row, + Arrays.toString(nonVectorizedGet), + Arrays.toString(vectorGet) + ), + Arrays.stream(nonVectorizedGet).mapToDouble(d -> (double) d).toArray(), + Arrays.stream(vectorGet).mapToDouble(d -> (double) d).toArray(), + 0.01 + ); + } else { + Assert.assertArrayEquals( + StringUtils.format( + "Results differed at row %s (%s : %s)", + row, + Arrays.toString(vectorGet), + Arrays.toString(nonVectorizedGet) + ), + vectorGet, + nonVectorizedGet + ); + } + } + catch (Throwable t) { + log.warn(t.getMessage()); + misMatch++; + } + vectorizedYielder = vectorizedYielder.next(vectorGet); + nonVectorizedYielder = nonVectorizedYielder.next(nonVectorizedGet); + row++; + } + try { + Assert.assertEquals("Expected no mismatched results", 0, misMatch); + Assert.assertTrue(vectorizedYielder.isDone()); + Assert.assertTrue(nonVectorizedYielder.isDone()); + } + catch (Throwable ignored) { + // the show must go on + } + } + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java b/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java index d6f4ed2bd876..275da27d07ec 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java +++ b/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java @@ -27,6 +27,8 @@ import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.UOE; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.ArrayList; @@ -49,6 +51,16 @@ public interface ApplyFunction */ String name(); + default boolean canVectorize(Expr.InputBindingTypes inputTypes, Expr lambda, List args) + { + return false; + } + + default VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, Expr lambda, List args) + { + throw new UOE("%s is not vectorized", name()); + } + /** * Apply {@link LambdaExpr} to argument list of {@link Expr} given a set of outer {@link Expr.ObjectBinding}. These * outer bindings will be used to form the scope for the bindings used to evaluate the {@link LambdaExpr}, which use diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java index 58cb5a08de87..dc52389de956 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java @@ -20,6 +20,12 @@ package org.apache.druid.math.expr; import org.apache.druid.java.util.common.guava.Comparators; +import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleDoubleLongBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleLongDoubleBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoublesBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongsBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.Objects; @@ -68,6 +74,82 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left < right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) < 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) < 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) < 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinLeqExpr extends BinaryEvalOpExprBase @@ -112,6 +194,83 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left <= right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) <= 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) <= 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) <= 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinGtExpr extends BinaryEvalOpExprBase @@ -156,6 +315,82 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left > right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) > 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) > 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) > 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinGeqExpr extends BinaryEvalOpExprBase @@ -200,6 +435,83 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left >= right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) >= 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) >= 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) >= 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinEqExpr extends BinaryEvalOpExprBase @@ -243,6 +555,83 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left == right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(left == right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(left == right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(left == right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinNeqExpr extends BinaryEvalOpExprBase @@ -286,6 +675,83 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left != right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(left != right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(left != right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(left != right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinAndExpr extends BinaryOpExprBase diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java index 21fadd490327..f56bdf033774 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java @@ -22,12 +22,18 @@ import com.google.common.math.LongMath; import com.google.common.primitives.Ints; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleDoubleLongBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleLongDoubleBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoublesBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongsBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; // math operators live here -class BinPlusExpr extends BinaryEvalOpExprBase +final class BinPlusExpr extends BinaryEvalOpExprBase { BinPlusExpr(String op, Expr left, Expr right) { @@ -43,24 +49,100 @@ protected BinaryOpExprBase copy(Expr left, Expr right) @Override protected ExprEval evalString(@Nullable String left, @Nullable String right) { - return ExprEval.of(NullHandling.nullToEmptyIfNeeded(left) - + NullHandling.nullToEmptyIfNeeded(right)); + return ExprEval.of(NullHandling.nullToEmptyIfNeeded(left) + NullHandling.nullToEmptyIfNeeded(right)); } @Override - protected final long evalLong(long left, long right) + protected long evalLong(long left, long right) { return left + right; } @Override - protected final double evalDouble(double left, double right) + protected double evalDouble(double left, double right) { return left + right; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left + right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left + right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left + (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left + right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } -class BinMinusExpr extends BinaryEvalOpExprBase +final class BinMinusExpr extends BinaryEvalOpExprBase { BinMinusExpr(String op, Expr left, Expr right) { @@ -74,19 +156,96 @@ protected BinaryOpExprBase copy(Expr left, Expr right) } @Override - protected final long evalLong(long left, long right) + protected long evalLong(long left, long right) { return left - right; } @Override - protected final double evalDouble(double left, double right) + protected double evalDouble(double left, double right) { return left - right; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left - right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left - right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left - (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left - right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } -class BinMulExpr extends BinaryEvalOpExprBase +final class BinMulExpr extends BinaryEvalOpExprBase { BinMulExpr(String op, Expr left, Expr right) { @@ -100,19 +259,96 @@ protected BinaryOpExprBase copy(Expr left, Expr right) } @Override - protected final long evalLong(long left, long right) + protected long evalLong(long left, long right) { return left * right; } @Override - protected final double evalDouble(double left, double right) + protected double evalDouble(double left, double right) { return left * right; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left * right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left * right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left * (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left * right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } -class BinDivExpr extends BinaryEvalOpExprBase +final class BinDivExpr extends BinaryEvalOpExprBase { BinDivExpr(String op, Expr left, Expr right) { @@ -126,16 +362,93 @@ protected BinaryOpExprBase copy(Expr left, Expr right) } @Override - protected final long evalLong(long left, long right) + protected long evalLong(long left, long right) { return left / right; } @Override - protected final double evalDouble(double left, double right) + protected double evalDouble(double left, double right) { return left / right; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left / right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left / right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left / (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left / right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinPowExpr extends BinaryEvalOpExprBase @@ -152,16 +465,93 @@ protected BinaryOpExprBase copy(Expr left, Expr right) } @Override - protected final long evalLong(long left, long right) + protected long evalLong(long left, long right) { return LongMath.pow(left, Ints.checkedCast(right)); } @Override - protected final double evalDouble(double left, double right) + protected double evalDouble(double left, double right) { return Math.pow(left, right); } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return LongMath.pow(left, Ints.checkedCast(right)); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.pow(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.pow(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.pow(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class BinModuloExpr extends BinaryEvalOpExprBase @@ -178,14 +568,91 @@ protected BinaryOpExprBase copy(Expr left, Expr right) } @Override - protected final long evalLong(long left, long right) + protected long evalLong(long left, long right) { return left % right; } @Override - protected final double evalDouble(double left, double right) + protected double evalDouble(double left, double right) { return left % right; } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left % right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left % right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left % (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left % right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } diff --git a/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java b/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java index ef090b5edd3a..7a7b80f11e7b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java @@ -23,6 +23,11 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.DoubleVectorExprEval; +import org.apache.druid.math.expr.vector.LongVectorExprEval; +import org.apache.druid.math.expr.vector.StringVectorExprEval; +import org.apache.druid.math.expr.vector.VectorExprEval; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.Arrays; @@ -133,6 +138,40 @@ public ExprEval eval(ObjectBinding bindings) return ExprEval.ofLong(value); } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return true; + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final long[] constant = new long[inputTypes.getMaxVectorSize()]; + final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; + if (value != null) { + Arrays.fill(constant, value); + } else { + Arrays.fill(nulls, true); + } + return new VectorExprProcessor() + { + private final VectorExprEval eval = + (VectorExprEval) new LongVectorExprEval(constant, nulls); + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + }; + } + @Override public boolean equals(Object o) { @@ -166,6 +205,36 @@ public ExprEval eval(ObjectBinding bindings) return ExprEval.ofLong(null); } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return true; + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final long[] constant = new long[inputTypes.getMaxVectorSize()]; + final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; + Arrays.fill(nulls, true); + return new VectorExprProcessor() + { + private final VectorExprEval eval = + (VectorExprEval) new LongVectorExprEval(constant, nulls); + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + }; + } + @Override public final int hashCode() { @@ -236,18 +305,16 @@ public int hashCode() } } -class StringExpr extends ConstantExpr +class DoubleExpr extends ConstantExpr { - @Nullable - private final String value; + private final Double value; - StringExpr(@Nullable String value) + DoubleExpr(Double value) { - super(ExprType.STRING); - this.value = NullHandling.emptyToNullIfNeeded(value); + super(ExprType.DOUBLE); + this.value = Preconditions.checkNotNull(value, "value"); } - @Nullable @Override public Object getLiteralValue() { @@ -257,22 +324,48 @@ public Object getLiteralValue() @Override public String toString() { - return value; + return String.valueOf(value); } @Override public ExprEval eval(ObjectBinding bindings) { - return ExprEval.of(value); + return ExprEval.ofDouble(value); } @Override - public String stringify() + public boolean canVectorize(InputBindingTypes inputTypes) { - // escape as javascript string since string literals are wrapped in single quotes - return value == null ? NULL_LITERAL : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(value)); + return true; } + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final double[] constant = new double[inputTypes.getMaxVectorSize()]; + final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; + if (value != null) { + Arrays.fill(constant, value); + } else { + Arrays.fill(nulls, true); + } + return new VectorExprProcessor() + { + private final VectorExprEval eval = + (VectorExprEval) new DoubleVectorExprEval(constant, nulls); + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + }; + } @Override public boolean equals(Object o) { @@ -282,7 +375,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - StringExpr that = (StringExpr) o; + DoubleExpr that = (DoubleExpr) o; return Objects.equals(value, that.value); } @@ -293,13 +386,69 @@ public int hashCode() } } -class StringArrayExpr extends ConstantExpr +class NullDoubleExpr extends NullNumericConstantExpr { - private final String[] value; + NullDoubleExpr() + { + super(ExprType.DOUBLE); + } - StringArrayExpr(String[] value) + @Override + public ExprEval eval(ObjectBinding bindings) { - super(ExprType.STRING_ARRAY); + return ExprEval.ofDouble(null); + } + + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return true; + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + final double[] constant = new double[inputTypes.getMaxVectorSize()]; + final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; + Arrays.fill(nulls, true); + return new VectorExprProcessor() + { + private final VectorExprEval eval = + (VectorExprEval) new DoubleVectorExprEval(constant, nulls); + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + }; + } + + @Override + public final int hashCode() + { + return NullDoubleExpr.class.hashCode(); + } + + @Override + public final boolean equals(Object obj) + { + return obj instanceof NullDoubleExpr; + } +} + +class DoubleArrayExpr extends ConstantExpr +{ + private final Double[] value; + + DoubleArrayExpr(Double[] value) + { + super(ExprType.DOUBLE_ARRAY); this.value = Preconditions.checkNotNull(value, "value"); } @@ -318,28 +467,16 @@ public String toString() @Override public ExprEval eval(ObjectBinding bindings) { - return ExprEval.ofStringArray(value); + return ExprEval.ofDoubleArray(value); } @Override public String stringify() { if (value.length == 0) { - return "[]"; + return "[]"; } - - return StringUtils.format( - "[%s]", - ARG_JOINER.join( - Arrays.stream(value) - .map(s -> s == null - ? NULL_LITERAL - // escape as javascript string since string literals are wrapped in single quotes - : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(s)) - ) - .iterator() - ) - ); + return StringUtils.format("%s", toString()); } @Override @@ -351,7 +488,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - StringArrayExpr that = (StringArrayExpr) o; + DoubleArrayExpr that = (DoubleArrayExpr) o; return Arrays.equals(value, that.value); } @@ -362,16 +499,18 @@ public int hashCode() } } -class DoubleExpr extends ConstantExpr +class StringExpr extends ConstantExpr { - private final Double value; + @Nullable + private final String value; - DoubleExpr(Double value) + StringExpr(@Nullable String value) { - super(ExprType.DOUBLE); - this.value = Preconditions.checkNotNull(value, "value"); + super(ExprType.STRING); + this.value = NullHandling.emptyToNullIfNeeded(value); } + @Nullable @Override public Object getLiteralValue() { @@ -381,68 +520,78 @@ public Object getLiteralValue() @Override public String toString() { - return String.valueOf(value); + return value; } @Override public ExprEval eval(ObjectBinding bindings) { - return ExprEval.ofDouble(value); + return ExprEval.of(value); } @Override - public boolean equals(Object o) + public boolean canVectorize(InputBindingTypes inputTypes) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DoubleExpr that = (DoubleExpr) o; - return Objects.equals(value, that.value); + return true; } @Override - public int hashCode() + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - return Objects.hash(value); - } -} + final String[] strings = new String[inputTypes.getMaxVectorSize()]; + Arrays.fill(strings, value); + return new VectorExprProcessor() + { + private final VectorExprEval eval = (VectorExprEval) new StringVectorExprEval(strings); -class NullDoubleExpr extends NullNumericConstantExpr -{ - NullDoubleExpr() - { - super(ExprType.DOUBLE); + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.STRING; + } + }; } @Override - public ExprEval eval(ObjectBinding bindings) + public String stringify() { - return ExprEval.ofDouble(null); + // escape as javascript string since string literals are wrapped in single quotes + return value == null ? NULL_LITERAL : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(value)); } @Override - public final int hashCode() + public boolean equals(Object o) { - return NullDoubleExpr.class.hashCode(); + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StringExpr that = (StringExpr) o; + return Objects.equals(value, that.value); } @Override - public final boolean equals(Object obj) + public int hashCode() { - return obj instanceof NullDoubleExpr; + return Objects.hash(value); } } -class DoubleArrayExpr extends ConstantExpr +class StringArrayExpr extends ConstantExpr { - private final Double[] value; + private final String[] value; - DoubleArrayExpr(Double[] value) + StringArrayExpr(String[] value) { - super(ExprType.DOUBLE_ARRAY); + super(ExprType.STRING_ARRAY); this.value = Preconditions.checkNotNull(value, "value"); } @@ -461,16 +610,28 @@ public String toString() @Override public ExprEval eval(ObjectBinding bindings) { - return ExprEval.ofDoubleArray(value); + return ExprEval.ofStringArray(value); } @Override public String stringify() { if (value.length == 0) { - return "[]"; + return "[]"; } - return StringUtils.format("%s", toString()); + + return StringUtils.format( + "[%s]", + ARG_JOINER.join( + Arrays.stream(value) + .map(s -> s == null + ? NULL_LITERAL + // escape as javascript string since string literals are wrapped in single quotes + : StringUtils.format("'%s'", StringEscapeUtils.escapeJavaScript(s)) + ) + .iterator() + ) + ); } @Override @@ -482,7 +643,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - DoubleArrayExpr that = (DoubleArrayExpr) o; + StringArrayExpr that = (StringArrayExpr) o; return Arrays.equals(value, that.value); } diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index 2a13be3f8459..bfa4e545e367 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -24,6 +24,7 @@ import com.google.common.collect.Sets; import org.apache.druid.annotations.SubclassesMustOverrideEqualsAndHashCode; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.ArrayList; @@ -40,6 +41,7 @@ public interface Expr { String NULL_LITERAL = "null"; Joiner ARG_JOINER = Joiner.on(", "); + /** * Indicates expression is a constant whose literal value can be extracted by {@link Expr#getLiteralValue()}, * making evaluating with arguments and bindings unecessary @@ -122,6 +124,7 @@ default String getBindingIfIdentifier() */ Expr visit(Shuttle shuttle); + /** * Examine the usage of {@link IdentifierExpr} children of an {@link Expr}, constructing a {@link BindingAnalysis} */ @@ -139,6 +142,16 @@ default ExprType getOutputType(InputBindingTypes inputTypes) return null; } + default boolean canVectorize(InputBindingTypes inputTypes) + { + return false; + } + + default VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + throw Exprs.cannotVectorize(this); + } + /** * Mechanism to supply input types for the bindings which will back {@link IdentifierExpr}, to use in the aid of * inferring the output type of an expression with {@link #getOutputType}. A null value means that either the binding @@ -148,6 +161,57 @@ interface InputBindingTypes { @Nullable ExprType getType(String name); + + default boolean areNumeric(List args) + { + boolean numeric = args.size() > 0; + for (Expr arg : args) { + ExprType argType = arg.getOutputType(this); + if (argType == null) { + numeric = false; + break; + } + numeric &= argType.isNumeric(); + } + return numeric; + } + + default boolean areNumeric(Expr... args) + { + boolean numeric = args.length > 0; + for (Expr arg : args) { + ExprType argType = arg.getOutputType(this); + if (argType == null) { + numeric = false; + break; + } + numeric &= argType.isNumeric(); + } + return numeric; + } + + default boolean canVectorize(List args) + { + boolean canVectorize = true; + for (Expr arg : args) { + canVectorize &= arg.canVectorize(this); + } + return canVectorize; + } + + default boolean canVectorize(Expr... args) + { + boolean canVectorize = true; + for (Expr arg : args) { + canVectorize &= arg.canVectorize(this); + } + return canVectorize; + } + } + + interface VectorInputBindingTypes extends InputBindingTypes + { + int getMaxVectorSize(); } /** @@ -162,6 +226,17 @@ interface ObjectBinding Object get(String name); } + interface VectorInputBinding extends VectorInputBindingTypes + { + T[] getObjectVector(String name); + + long[] getLongVector(String name); + double[] getDoubleVector(String name); + boolean[] getNullVector(String name); + + int getCurrentVectorSize(); + } + /** * Mechanism to inspect an {@link Expr}, implementing a {@link Visitor} allows visiting all children of an * {@link Expr} diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 1c02186296be..8caa33db1861 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -121,6 +121,20 @@ public static ExprEval bestEffortOf(@Nullable Object val) return new StringExprEval(val == null ? null : String.valueOf(val)); } + @Nullable + public static Number computeNumber(@Nullable String value) + { + Number rv; + Long v = GuavaUtils.tryParseLong(value); + // Do NOT use ternary operator here, because it makes Java to convert Long to Double + if (v != null) { + rv = v; + } else { + rv = Doubles.tryParse(value); + } + return rv; + } + // Cached String values private boolean stringValueCached = false; @Nullable @@ -496,7 +510,7 @@ private double computeDouble() } @Nullable - private Number computeNumber() + Number computeNumber() { if (value == null) { return null; @@ -505,17 +519,8 @@ private Number computeNumber() // Optimization for non-null case. return numericVal; } - Number rv; - Long v = GuavaUtils.tryParseLong(value); - // Do NOT use ternary operator here, because it makes Java to convert Long to Double - if (v != null) { - rv = v; - } else { - rv = Doubles.tryParse(value); - } - - numericVal = rv; - return rv; + numericVal = computeNumber(value); + return numericVal; } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java b/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java index 3f69f6e0b7e4..617499ed734b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprListenerImpl.java @@ -73,10 +73,10 @@ public void exitUnaryOpExpr(ExprParser.UnaryOpExprContext ctx) int opCode = ((TerminalNode) ctx.getChild(0)).getSymbol().getType(); switch (opCode) { case ExprParser.MINUS: - nodes.put(ctx, new UnaryMinusExpr((Expr) nodes.get(ctx.getChild(1)))); + nodes.put(ctx, new UnaryMinusExpr(ctx.getChild(0).getText(), (Expr) nodes.get(ctx.getChild(1)))); break; case ExprParser.NOT: - nodes.put(ctx, new UnaryNotExpr((Expr) nodes.get(ctx.getChild(1)))); + nodes.put(ctx, new UnaryNotExpr(ctx.getChild(0).getText(), (Expr) nodes.get(ctx.getChild(1)))); break; default: throw new RE("Unrecognized unary operator %s", ctx.getChild(0).getText()); diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprType.java b/core/src/main/java/org/apache/druid/math/expr/ExprType.java index 3b9108de9216..c836ef8781e9 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprType.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprType.java @@ -37,6 +37,7 @@ public enum ExprType LONG_ARRAY, STRING_ARRAY; + public boolean isNumeric() { return isNumeric(this); @@ -74,6 +75,27 @@ public static ExprType fromValueType(@Nullable ValueType valueType) } } + + public static ValueType toValueType(ExprType exprType) + { + switch (exprType) { + case LONG: + return ValueType.LONG; + case LONG_ARRAY: + return ValueType.LONG_ARRAY; + case DOUBLE: + return ValueType.DOUBLE; + case DOUBLE_ARRAY: + return ValueType.DOUBLE_ARRAY; + case STRING: + return ValueType.STRING; + case STRING_ARRAY: + return ValueType.STRING_ARRAY; + default: + throw new ISE("Unsupported expression type[%s]", exprType); + } + } + public static boolean isNumeric(ExprType type) { return LONG.equals(type) || DOUBLE.equals(type); diff --git a/core/src/main/java/org/apache/druid/math/expr/Exprs.java b/core/src/main/java/org/apache/druid/math/expr/Exprs.java index b1a34386eb6e..13783ee5d6eb 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Exprs.java +++ b/core/src/main/java/org/apache/druid/math/expr/Exprs.java @@ -20,6 +20,7 @@ package org.apache.druid.math.expr; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.UOE; import java.util.ArrayList; import java.util.List; @@ -28,6 +29,21 @@ public class Exprs { + public static UnsupportedOperationException cannotVectorize(Expr expr) + { + return new UOE("Unable to vectorize expression:[%s]", expr.stringify()); + } + + public static UnsupportedOperationException cannotVectorize(Function function) + { + return new UOE("Unable to vectorize function:[%s]", function.name()); + } + + public static UnsupportedOperationException cannotVectorize() + { + return new UOE("Unable to vectorize expression"); + } + /** * Decomposes any expr into a list of exprs that, if ANDed together, are equivalent to the input expr. * diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 2e27aab84ae6..085d02cfa941 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -25,6 +25,17 @@ import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.UOE; +import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleDoubleLongBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleLongDoubleBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoublesBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongDoubleUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongsBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.StringLongUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -112,6 +123,16 @@ default boolean hasArrayOutput() @Nullable ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args); + default boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return false; + } + + default VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + throw new UOE("%s is not vectorized", name()); + } + /** * Base class for a single variable input {@link Function} implementation */ @@ -517,6 +538,52 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) return ExprEval.of(retVal); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return args.size() == 1 && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + ExprType parseType = args.get(0).getOutputType(inputTypes); + VectorExprProcessor processor; + + if (args.size() == 1) { + VectorExprProcessor inputProcessor = args.get(0).buildVectorized(inputTypes); + // if its already a long, just pass it through + if (ExprType.LONG.equals(inputProcessor.getOutputType())) { + processor = inputProcessor; + } else { + // else, always convert the argument to a string, who cares what it is + processor = new StringLongUnivariateFunctionVectorProcessor( + CastToTypeVectorProcessor.castToType(inputProcessor, ExprType.STRING), + inputTypes.getMaxVectorSize() + ) + { + @Override + public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) + { + try { + longs[i] = Long.parseLong(strings[i], 10); + outputNulls[i] = false; + } + catch (NumberFormatException e) { + longs[i] = 0L; + outputNulls[i] = NullHandling.sqlCompatible(); + } + } + }; + } + } else { + // not yet implemented, how did we get here + throw Exprs.cannotVectorize(this); + } + + return (VectorExprProcessor) processor; + } } class Pi implements Function @@ -615,6 +682,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.atan(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.atan(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.atan(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Cbrt extends DoubleUnivariateMathFunction @@ -660,6 +767,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.cos(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cos(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cos(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Cosh extends DoubleUnivariateMathFunction @@ -675,6 +822,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.cosh(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cosh(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cosh(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Cot extends DoubleUnivariateMathFunction @@ -690,6 +877,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.cos(param) / Math.sin(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cos(input) / Math.sin(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cos(input) / Math.sin(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Div extends BivariateMathFunction @@ -964,6 +1191,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.sin(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.sin(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.sin(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Sinh extends DoubleUnivariateMathFunction @@ -979,6 +1246,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.sinh(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.sinh(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.sinh(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Sqrt extends DoubleUnivariateMathFunction @@ -1009,6 +1316,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.tan(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.tan(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.tan(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class Tanh extends DoubleUnivariateMathFunction @@ -1024,6 +1371,46 @@ protected ExprEval eval(double param) { return ExprEval.of(Math.tanh(param)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + Expr arg = args.get(0); + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongDoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.tanh(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.tanh(input); + } + }; + } + return (VectorExprProcessor) processor; + } } class ToDegrees extends DoubleUnivariateMathFunction @@ -1150,6 +1537,83 @@ protected ExprEval eval(double x, double y) { return ExprEval.of(Math.max(x, y)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + final ExprType leftType = args.get(0).getOutputType(inputTypes); + final ExprType rightType = args.get(1).getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Math.max(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.max(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.max(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.max(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class Min extends BivariateMathFunction @@ -1171,6 +1635,82 @@ protected ExprEval eval(double x, double y) { return ExprEval.of(Math.min(x, y)); } + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + final ExprType leftType = args.get(0).getOutputType(inputTypes); + final ExprType rightType = args.get(1).getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongsBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Math.min(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.min(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.min(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoublesBivariateFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.min(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class NextAfter extends DoubleBivariateMathFunction @@ -1295,6 +1835,23 @@ public ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args } return null; } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return args.get(0).canVectorize(inputTypes) && (args.get(1).isLiteral() || args.get(1).canVectorize(inputTypes)); + } + + @Override + public VectorExprProcessor asVectorProcessor( + Expr.VectorInputBindingTypes inputTypes, List args + ) + { + return CastToTypeVectorProcessor.castToType( + args.get(0).buildVectorized(inputTypes), + ExprType.valueOf(StringUtils.toUpperCase(args.get(1).getLiteralValue().toString())) + ); + } } class GreatestFunc extends ReduceFunction diff --git a/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java b/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java index e81d5bafd2cc..5323c87d1928 100644 --- a/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.List; @@ -76,6 +77,18 @@ public Expr getExpr() return expr; } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return expr.canVectorize(inputTypes); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + return expr.buildVectorized(inputTypes); + } + @Override public ExprEval eval(ObjectBinding bindings) { @@ -170,6 +183,18 @@ public ExprEval eval(ObjectBinding bindings) return function.apply(args, bindings); } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return function.canVectorize(inputTypes, args); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + return function.asVectorProcessor(inputTypes, args); + } + @Override public String stringify() { @@ -288,6 +313,20 @@ public ExprEval eval(ObjectBinding bindings) return function.apply(lambdaExpr, argsExpr, bindings); } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return function.canVectorize(inputTypes, lambdaExpr, argsExpr) && + lambdaExpr.canVectorize(inputTypes) && + argsExpr.stream().allMatch(expr -> expr.canVectorize(inputTypes)); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + return function.asVectorProcessor(inputTypes, lambdaExpr, argsExpr); + } + @Override public String stringify() { diff --git a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java index 437370641c61..d9faab0a43f5 100644 --- a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java @@ -21,6 +21,11 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.DoubleVectorExprEval; +import org.apache.druid.math.expr.vector.LongVectorExprEval; +import org.apache.druid.math.expr.vector.StringVectorExprEval; +import org.apache.druid.math.expr.vector.VectorExprEval; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.Objects; @@ -32,8 +37,8 @@ */ class IdentifierExpr implements Expr { - private final String identifier; - private final String binding; + final String identifier; + final String binding; /** * Construct a identifier expression for a {@link LambdaExpr}, where the {@link #identifier} is equal to @@ -113,6 +118,12 @@ public ExprType getOutputType(InputBindingTypes inputTypes) return inputTypes.getType(binding); } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return true; + } + @Override public ExprEval eval(ObjectBinding bindings) { @@ -138,6 +149,42 @@ public Expr visit(Shuttle shuttle) return shuttle.visit(this); } + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + switch (inputTypes.getType(binding)) { + case LONG: + return new IdentifierVectorProcessor(ExprType.LONG) + { + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return new LongVectorExprEval(bindings.getLongVector(binding), bindings.getNullVector(binding)); + } + }; + case DOUBLE: + return new IdentifierVectorProcessor(ExprType.DOUBLE) + { + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return new DoubleVectorExprEval(bindings.getDoubleVector(binding), bindings.getNullVector(binding)); + } + }; + case STRING: + return new IdentifierVectorProcessor(ExprType.STRING) + { + @Override + public VectorExprEval evalVector(VectorInputBinding bindings) + { + return new StringVectorExprEval(bindings.getObjectVector(binding)); + } + }; + default: + throw Exprs.cannotVectorize(this); + } + } + @Override public boolean equals(Object o) { @@ -157,3 +204,20 @@ public int hashCode() return Objects.hash(identifier); } } + +abstract class IdentifierVectorProcessor implements VectorExprProcessor +{ + private final ExprType outputType; + + public IdentifierVectorProcessor(ExprType outputType) + { + this.outputType = outputType; + } + + @Override + public ExprType getOutputType() + { + return outputType; + } +} + diff --git a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java index 3d68430ea65b..813eec85c887 100644 --- a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java @@ -23,6 +23,9 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.DoubleUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.Objects; @@ -32,10 +35,12 @@ */ abstract class UnaryExpr implements Expr { + final String op; final Expr expr; - UnaryExpr(Expr expr) + UnaryExpr(String op, Expr expr) { + this.op = op; this.expr = expr; } @@ -91,19 +96,31 @@ public int hashCode() { return Objects.hash(expr); } + + @Override + public String stringify() + { + return StringUtils.format("%s%s", op, expr.stringify()); + } + + @Override + public String toString() + { + return StringUtils.format("%s%s", op, expr); + } } class UnaryMinusExpr extends UnaryExpr { - UnaryMinusExpr(Expr expr) + UnaryMinusExpr(String op, Expr expr) { - super(expr); + super(op, expr); } @Override UnaryExpr copy(Expr expr) { - return new UnaryMinusExpr(expr); + return new UnaryMinusExpr(op, expr); } @Override @@ -123,29 +140,58 @@ public ExprEval eval(ObjectBinding bindings) } @Override - public String stringify() + public boolean canVectorize(InputBindingTypes inputTypes) { - return StringUtils.format("-%s", expr.stringify()); + return inputTypes.areNumeric(expr) && expr.canVectorize(inputTypes); } @Override - public String toString() + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - return StringUtils.format("-%s", expr); + final ExprType inputType = expr.getOutputType(inputTypes); + if (inputType == null) { + throw Exprs.cannotVectorize(this); + } + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongUnivariateFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) + { + @Override + public long apply(long input) + { + return -input; + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleUnivariateFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) + { + @Override + public double apply(double input) + { + return -input; + } + }; + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } class UnaryNotExpr extends UnaryExpr { - UnaryNotExpr(Expr expr) + UnaryNotExpr(String op, Expr expr) { - super(expr); + super(op, expr); } @Override UnaryExpr copy(Expr expr) { - return new UnaryNotExpr(expr); + return new UnaryNotExpr(op, expr); } @Override @@ -160,18 +206,6 @@ public ExprEval eval(ObjectBinding bindings) return ExprEval.of(!ret.asBoolean(), retType); } - @Override - public String stringify() - { - return StringUtils.format("!%s", expr.stringify()); - } - - @Override - public String toString() - { - return StringUtils.format("!%s", expr); - } - @Nullable @Override public ExprType getOutputType(InputBindingTypes inputTypes) diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..db577be83104 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; + +public abstract class BivariateFunctionVectorProcessor implements VectorExprProcessor +{ + final VectorExprProcessor left; + final VectorExprProcessor right; + final int maxVectorSize; + final boolean[] outNulls; + final TOutput outValues; + + protected BivariateFunctionVectorProcessor( + VectorExprProcessor left, + VectorExprProcessor right, + int maxVectorSize, + TOutput outValues + ) + { + this.left = left; + this.right = right; + this.maxVectorSize = maxVectorSize; + this.outNulls = new boolean[maxVectorSize]; + this.outValues = outValues; + } + + @Override + public final VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + final VectorExprEval lhs = left.evalVector(bindings); + final VectorExprEval rhs = right.evalVector(bindings); + + final int currentSize = bindings.getCurrentVectorSize(); + final boolean[] leftNulls = lhs.getNullVector(); + final boolean[] rightNulls = rhs.getNullVector(); + final boolean hasLeftNulls = leftNulls != null; + final boolean hasRightNulls = rightNulls != null; + final boolean hasNulls = hasLeftNulls || hasRightNulls; + + final TLeftInput leftInput = lhs.values(); + final TRightInput rightInput = rhs.values(); + + if (hasNulls) { + for (int i = 0; i < currentSize; i++) { + outNulls[i] = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); + if (!outNulls[i]) { + processIndex(leftInput, rightInput, i); + } else { + processNull(i); + } + } + } else { + for (int i = 0; i < currentSize; i++) { + processIndex(leftInput, rightInput, i); + outNulls[i] = false; + } + } + return asEval(); + } + + abstract void processIndex(TLeftInput leftInput, TRightInput rightInput, int i); + + abstract void processNull(int i); + + abstract VectorExprEval asEval(); +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java new file mode 100644 index 000000000000..a790101286cf --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; + +public class CastToDoubleVectorProcessor extends CastToTypeVectorProcessor +{ + public CastToDoubleVectorProcessor(VectorExprProcessor delegate) + { + super(delegate); + } + + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + VectorExprEval result = delegate.evalVector(bindings); + return new DoubleVectorExprEval(result.getDoubleVector(), result.getNullVector()); + } + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java new file mode 100644 index 000000000000..8a725b377126 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; + +public class CastToLongVectorProcessor extends CastToTypeVectorProcessor +{ + public CastToLongVectorProcessor(VectorExprProcessor delegate) + { + super(delegate); + } + + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + VectorExprEval result = delegate.evalVector(bindings); + return new LongVectorExprEval(result.getLongVector(), result.getNullVector()); + } + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java new file mode 100644 index 000000000000..abf570def8d8 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; + +public class CastToStringVectorProcessor extends CastToTypeVectorProcessor +{ + public CastToStringVectorProcessor(VectorExprProcessor delegate) + { + super(delegate); + } + + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + VectorExprEval result = delegate.evalVector(bindings); + return new StringVectorExprEval(result.asObjectVector(ExprType.STRING)); + } + + @Override + public ExprType getOutputType() + { + return ExprType.STRING; + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java new file mode 100644 index 000000000000..a9a04cdc9a82 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Exprs; + +public abstract class CastToTypeVectorProcessor implements VectorExprProcessor +{ + protected final VectorExprProcessor delegate; + + protected CastToTypeVectorProcessor(VectorExprProcessor delegate) + { + this.delegate = delegate; + } + + public static VectorExprProcessor castToType(VectorExprProcessor delegate, ExprType type) + { + final VectorExprProcessor caster; + if (delegate.getOutputType() == type) { + caster = delegate; + } else { + switch (type) { + case STRING: + caster = new CastToStringVectorProcessor(delegate); + break; + case LONG: + caster = new CastToLongVectorProcessor(delegate); + break; + case DOUBLE: + caster = new CastToDoubleVectorProcessor(delegate); + break; + default: + throw Exprs.cannotVectorize(); + } + } + return (VectorExprProcessor) caster; + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..65ecde018a3e --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class DoubleDoubleLongBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +{ + public DoubleDoubleLongBivariateFunctionVectorProcessor( + VectorExprProcessor left, + VectorExprProcessor right, + int maxVectorSize + ) + { + super(left, right, maxVectorSize, new double[maxVectorSize]); + } + + public abstract double apply(double left, long right); + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + + @Override + final void processIndex(double[] leftInput, long[] rightInput, int i) + { + outValues[i] = apply(leftInput[i], rightInput[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new DoubleVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..f828a673d24a --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class DoubleLongDoubleBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +{ + public DoubleLongDoubleBivariateFunctionVectorProcessor( + VectorExprProcessor left, + VectorExprProcessor right, + int maxVectorSize + ) + { + super(left, right, maxVectorSize, new double[maxVectorSize]); + } + + public abstract double apply(long left, double right); + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + + @Override + final void processIndex(long[] leftInput, double[] rightInput, int i) + { + outValues[i] = apply(leftInput[i], rightInput[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new DoubleVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..b275638ced3e --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class DoubleUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorProcessor +{ + public DoubleUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + { + super(processor, maxVectorSize, new double[maxVectorSize]); + } + + public abstract double apply(double input); + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + + @Override + final void processIndex(double[] longs, int i) + { + outValues[i] = apply(longs[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new DoubleVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java new file mode 100644 index 000000000000..ac8d2acc0aea --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.ExprType; + +import java.util.Arrays; + +public class DoubleVectorExprEval extends VectorExprEval +{ + public DoubleVectorExprEval(double[] values, boolean[] nulls) + { + super(values, nulls); + } + + @Override + public ExprType getType() + { + return ExprType.DOUBLE; + } + + @Override + public double[] values() + { + return values; + } + + @Override + public long[] getLongVector() + { + return Arrays.stream(values).mapToLong(d -> (long) d).toArray(); + } + + @Override + public double[] getDoubleVector() + { + return values; + } + + @Override + public E asObjectVector(ExprType type) + { + switch (type) { + case STRING: + return (E) Arrays.stream(values).mapToObj(String::valueOf).toArray(String[]::new); + default: + throw new IAE("Cannot convert %s to %s object vector", getType(), type); + } + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..bd17a8f1d410 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class DoublesBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +{ + public DoublesBivariateFunctionVectorProcessor( + VectorExprProcessor left, + VectorExprProcessor right, + int maxVectorSize + ) + { + super(left, right, maxVectorSize, new double[maxVectorSize]); + } + + public abstract double apply(double left, double right); + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + + @Override + final void processIndex(double[] leftInput, double[] rightInput, int i) + { + outValues[i] = apply(leftInput[i], rightInput[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new DoubleVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..e59e54fb31c8 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class LongDoubleUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorProcessor +{ + public LongDoubleUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + { + super(processor, maxVectorSize, new double[maxVectorSize]); + } + + public abstract double apply(long input); + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + + @Override + final void processIndex(long[] longs, int i) + { + outValues[i] = apply(longs[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new DoubleVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..b54a021507d8 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class LongUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorProcessor +{ + public LongUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + { + super(processor, maxVectorSize, new long[maxVectorSize]); + } + + public abstract long apply(long input); + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + + @Override + final void processIndex(long[] longs, int i) + { + outValues[i] = apply(longs[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new LongVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java new file mode 100644 index 000000000000..8932342d5593 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.ExprType; + +import javax.annotation.Nullable; +import java.util.Arrays; + +public class LongVectorExprEval extends VectorExprEval +{ + public LongVectorExprEval(long[] values, @Nullable boolean[] nulls) + { + super(values, nulls); + } + + @Override + public ExprType getType() + { + return ExprType.LONG; + } + + @Override + public long[] getLongVector() + { + return values; + } + + @Override + public double[] getDoubleVector() + { + return Arrays.stream(values).asDoubleStream().toArray(); + } + + @Override + public E asObjectVector(ExprType type) + { + switch (type) { + case STRING: + return (E) Arrays.stream(values).mapToObj(String::valueOf).toArray(String[]::new); + default: + throw new IAE("Cannot convert %s to %s object vector", getType(), type); + } + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..9b294d6d1f50 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class LongsBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +{ + public LongsBivariateFunctionVectorProcessor( + VectorExprProcessor left, + VectorExprProcessor right, + int maxVectorSize + ) + { + super(left, right, maxVectorSize, new long[maxVectorSize]); + } + + public abstract long apply(long left, long right); + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + + @Override + final void processIndex(long[] leftInput, long[] rightInput, int i) + { + outValues[i] = apply(leftInput[i], rightInput[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new LongVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..b50dc379906e --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class LongsDoubleBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +{ + public LongsDoubleBivariateFunctionVectorProcessor( + VectorExprProcessor left, + VectorExprProcessor right, + int maxVectorSize + ) + { + super(left, right, maxVectorSize, new double[maxVectorSize]); + } + + public abstract double apply(long left, long right); + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + + @Override + final void processIndex(long[] leftInput, long[] rightInput, int i) + { + outValues[i] = apply(leftInput[i], rightInput[i]); + } + + @Override + void processNull(int i) + { + outValues[i] = 0; + } + + @Override + final VectorExprEval asEval() + { + return new DoubleVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..512324c5a82c --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.ExprType; + +public abstract class StringLongUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorObjectProcessor +{ + public StringLongUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + { + super(processor, maxVectorSize, new long[maxVectorSize]); + } + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + + @Override + public final VectorExprEval asEval() + { + return new LongVectorExprEval(outValues, outNulls); + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java new file mode 100644 index 000000000000..19b80c3788b0 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprType; + +import javax.annotation.Nullable; + +public class StringVectorExprEval extends VectorExprEval +{ + @Nullable + private long[] longs; + @Nullable + private double[] doubles; + + @Nullable + private boolean[] numericNulls; + + public StringVectorExprEval(String[] values) + { + super(values, null); + } + + private void computeNumbers() + { + longs = new long[values.length]; + doubles = new double[values.length]; + numericNulls = new boolean[values.length]; + for (int i = 0; i < values.length; i++) { + Number n = ExprEval.computeNumber(values[i]); + if (n != null) { + longs[i] = n.longValue(); + doubles[i] = n.doubleValue(); + numericNulls[i] = false; + } else { + longs[i] = 0L; + doubles[i] = 0.0; + numericNulls[i] = NullHandling.sqlCompatible(); + } + } + } + + @Nullable + @Override + public boolean[] getNullVector() + { + if (numericNulls == null) { + computeNumbers(); + } + return numericNulls; + } + + @Override + public ExprType getType() + { + return ExprType.STRING; + } + + @Override + public long[] getLongVector() + { + if (longs == null) { + computeNumbers(); + } + return longs; + } + + @Override + public double[] getDoubleVector() + { + if (doubles == null) { + computeNumbers(); + } + return doubles; + } + + @Override + public E getObjectVector() + { + return (E) values; + } + + @Override + public E asObjectVector(ExprType type) + { + switch (type) { + case STRING: + return (E) values; + default: + throw new IAE("Cannot convert %s to %s object vector", getType(), type); + } + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java new file mode 100644 index 000000000000..2b37f86d3f29 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; + +public abstract class UnivariateFunctionVectorObjectProcessor implements VectorExprProcessor +{ + final VectorExprProcessor processor; + final int maxVectorSize; + final boolean[] outNulls; + final TOutput outValues; + + public UnivariateFunctionVectorObjectProcessor( + VectorExprProcessor processor, + int maxVectorSize, + TOutput outValues + ) + { + this.processor = processor; + this.maxVectorSize = maxVectorSize; + this.outNulls = new boolean[maxVectorSize]; + this.outValues = outValues; + } + + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + final VectorExprEval lhs = processor.evalVector(bindings); + + final int currentSize = bindings.getCurrentVectorSize(); + + final TInput input = lhs.values(); + + for (int i = 0; i < currentSize; i++) { + processIndex(input, outValues, outNulls, i); + } + return asEval(); + } + + public abstract void processIndex(TInput input, TOutput output, boolean[] outputNulls, int i); + + public abstract VectorExprEval asEval(); +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java new file mode 100644 index 000000000000..1b5aca533dfe --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; + +public abstract class UnivariateFunctionVectorProcessor implements VectorExprProcessor +{ + final VectorExprProcessor processor; + final int maxVectorSize; + final boolean[] outNulls; + final TOutput outValues; + + public UnivariateFunctionVectorProcessor( + VectorExprProcessor processor, + int maxVectorSize, + TOutput outValues + ) + { + this.processor = processor; + this.maxVectorSize = maxVectorSize; + this.outNulls = new boolean[maxVectorSize]; + this.outValues = outValues; + } + + @Override + public final VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + final VectorExprEval lhs = processor.evalVector(bindings); + + final int currentSize = bindings.getCurrentVectorSize(); + final boolean[] inputNulls = lhs.getNullVector(); + final boolean hasNulls = inputNulls != null; + + final TInput input = lhs.values(); + + if (hasNulls) { + for (int i = 0; i < currentSize; i++) { + outNulls[i] = inputNulls[i]; + if (!outNulls[i]) { + processIndex(input, i); + } else { + processNull(i); + } + } + } else { + for (int i = 0; i < currentSize; i++) { + outNulls[i] = false; + processIndex(input, i); + } + } + return asEval(); + } + + abstract void processIndex(TInput input, int i); + + abstract void processNull(int i); + + abstract VectorExprEval asEval(); +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java new file mode 100644 index 000000000000..17b190410ce3 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.ExprType; + +import javax.annotation.Nullable; +import java.lang.reflect.Array; + +public abstract class VectorExprEval +{ + final T values; + @Nullable + final boolean[] nulls; + + public VectorExprEval(T values, @Nullable boolean[] nulls) + { + this.values = values; + this.nulls = nulls; + } + + public T values() + { + return values; + } + + @Nullable + public Object get(int index) + { + if (nulls == null || NullHandling.replaceWithDefault() || !nulls[index]) { + return Array.get(values, index); + } + return null; + } + + @Nullable + public boolean[] getNullVector() + { + return nulls; + } + + public abstract ExprType getType(); + + public abstract long[] getLongVector(); + + public abstract double[] getDoubleVector(); + + public E getObjectVector() + { + // non-primitives should implement this + throw new IllegalArgumentException("Object vector not available"); + } + + public abstract E asObjectVector(ExprType type); +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java new file mode 100644 index 000000000000..b17933b6a04c --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; + +public interface VectorExprProcessor +{ + VectorExprEval evalVector(Expr.VectorInputBinding bindings); + + ExprType getOutputType(); +} diff --git a/core/src/test/java/org/apache/druid/math/expr/ExprTest.java b/core/src/test/java/org/apache/druid/math/expr/ExprTest.java index 6dfa61d6d186..21239e91bdde 100644 --- a/core/src/test/java/org/apache/druid/math/expr/ExprTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/ExprTest.java @@ -120,13 +120,13 @@ public void testEqualsContractForApplyFunctionExpr() @Test public void testEqualsContractForUnaryNotExpr() { - EqualsVerifier.forClass(UnaryNotExpr.class).usingGetClass().verify(); + EqualsVerifier.forClass(UnaryNotExpr.class).withIgnoredFields("op").usingGetClass().verify(); } @Test public void testEqualsContractForUnaryMinusExpr() { - EqualsVerifier.forClass(UnaryMinusExpr.class).usingGetClass().verify(); + EqualsVerifier.forClass(UnaryMinusExpr.class).withIgnoredFields("op").usingGetClass().verify(); } @Test diff --git a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java index bd755ba7e0ec..3c5c214de657 100644 --- a/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -468,7 +468,7 @@ public void testGreatest() { // Same types assertExpr("greatest(y, 0)", 2L); - assertExpr("greatest(34.0, z, 5.0, 767.0", 767.0); + assertExpr("greatest(34.0, z, 5.0, 767.0)", 767.0); assertExpr("greatest('B', x, 'A')", "foo"); // Different types @@ -496,7 +496,7 @@ public void testLeast() { // Same types assertExpr("least(y, 0)", 0L); - assertExpr("least(34.0, z, 5.0, 767.0", 3.1); + assertExpr("least(34.0, z, 5.0, 767.0)", 3.1); assertExpr("least('B', x, 'A')", "A"); // Different types diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java new file mode 100644 index 000000000000..4335f9e2571e --- /dev/null +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.NonnullPair; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.math.expr.vector.VectorExprEval; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.Assert; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.BooleanSupplier; +import java.util.function.DoubleSupplier; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +/** + * randomize inputs to various vector expressions and make sure the results match nonvectorized expressions + * + * this is not a replacement for correctness tests, but will ensure that vectorized and non-vectorized expression + * evaluation is at least self consistent... + */ +public class VectorExprSanityTest extends InitializedNullHandlingTest +{ + private static final int NUM_ITERATIONS = 100; + private static final int VECTOR_SIZE = 512; + + final Map types = ImmutableMap.builder() + .put("l1", ExprType.LONG) + .put("l2", ExprType.LONG) + .put("d1", ExprType.DOUBLE) + .put("d2", ExprType.DOUBLE) + .put("s1", ExprType.STRING) + .put("s2", ExprType.STRING) + .build(); + + @Test + public void testUnaryOperators() + { + final String[] functions = new String[]{"-"}; + final String[] templates = new String[]{"%sd1", "%sl1"}; + + testFunctions(types, templates, functions); + } + + @Test + public void testBinaryOperators() + { + final String[] functions = new String[]{"+", "-", "*", "/", "^", "%", ">", ">=", "<", "<=", "==", "!="}; + final String[] templates = new String[]{ + "d1 %s d2", + "d1 %s l1", + "l1 %s d1", + "l1 %s l2", + "1 %s l1", + "1.1 %s d1", + "l1 %s 1", + "d1 %s 1.1" + }; + + testFunctions(types, templates, functions); + } + + @Test + public void testBinaryOperatorTrees() + { + final String[][] argsArrays = new String[][]{new String[]{"+", "-"}, new String[]{"*", "/"}}; + final String[] templates = new String[]{ + "(d1 %s d2) %s d2", + "(d1 %s d2) %s l2", + "(d1 %s l1) %s d2", + "(d1 %s l1) %s l2", + "(l1 %s d1) %s d2", + "(l1 %s d1) %s l2", + "(l1 %s l2) %s l2", + "(1 %s l1) %s d2", + "(1.1 %s d1) %s d2", + "(1.1 %s d1) %s l2", + "(l1 %s 1) %s l2", + "(l1 %s 1) %s d2", + "(d1 %s 1.1) %s d2", + "(d1 %s 1.1) %s l1" + }; + + testFunctions(types, templates, argsArrays); + } + + @Test + public void testUnivariateFunctions() + { + final String[] functions = new String[]{"parse_long"}; + final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testUnivariateMathFunctions() + { + final String[] functions = new String[]{"atan", "cos", "cosh", "cot", "sin", "sinh", "tan", "tanh"}; + final String[] templates = new String[]{"%s(l1)", "%s(d1)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testBivariateMathFunctions() + { + final String[] functions = new String[]{"max", "min"}; + final String[] templates = new String[]{"%s(d1, d2)", "%s(d1, l1)", "%s(l1, d1)", "%s(l1, l2)"}; + testFunctions(types, templates, functions); + } + + static void testFunctions(Map types, String[] templates, String[] args) + { + for (String template : templates) { + for (String arg : args) { + String expr = StringUtils.format(template, arg); + testExpression(expr, types); + } + } + } + + static void testFunctions(Map types, String[] templates, String[][] argsArrays) + { + for (String template : templates) { + for (Object[] args : argsArrays) { + String expr = StringUtils.format(template, args); + testExpression(expr, types); + } + } + } + + static void testExpression(String expr, Map types) + { + Expr parsed = Parser.parse(expr, ExprMacroTable.nil()); + + NonnullPair bindings; + for (int iterations = 0; iterations < NUM_ITERATIONS; iterations++) { + bindings = makeRandomizedBindings(VECTOR_SIZE, types); + testExpressionWithBindings(expr, parsed, bindings); + } + bindings = makeSequentialBinding(VECTOR_SIZE, types); + testExpressionWithBindings(expr, parsed, bindings); + } + + private static void testExpressionWithBindings( + String expr, + Expr parsed, + NonnullPair bindings + ) + { + Assert.assertTrue(parsed.canVectorize(bindings.rhs)); + ExprType outputType = parsed.getOutputType(bindings.rhs); + VectorExprEval vectorEval = parsed.buildVectorized(bindings.rhs).evalVector(bindings.rhs); + Assert.assertEquals(outputType, vectorEval.getType()); + for (int i = 0; i < VECTOR_SIZE; i++) { + ExprEval eval = parsed.eval(bindings.lhs[i]); + if (!eval.isNumericNull()) { + Assert.assertEquals(outputType, eval.type()); + } + Assert.assertEquals(StringUtils.format("Values do not match for row %s for expression %s", i, expr), eval.value(), vectorEval.get(i)); + } + } + + static NonnullPair makeRandomizedBindings(int vectorSize, Map types) + { + + final ThreadLocalRandom r = ThreadLocalRandom.current(); + return makeBindings( + vectorSize, + types, + () -> r.nextLong(Integer.MAX_VALUE - 1), + r::nextDouble, + r::nextBoolean, + () -> String.valueOf(r.nextInt()) + ); + } + + static NonnullPair makeSequentialBinding(int vectorSize, Map types) + { + + return makeBindings( + vectorSize, + types, + new LongSupplier() + { + int counter = 1; + @Override + public long getAsLong() + { + return counter++; + } + }, + new DoubleSupplier() + { + int counter = 1; + @Override + public double getAsDouble() + { + return counter++; + } + }, + () -> ThreadLocalRandom.current().nextBoolean(), + new Supplier() + { + int counter = 1; + @Override + public String get() + { + return String.valueOf(counter++); + } + } + ); + } + + static NonnullPair makeBindings( + int vectorSize, + Map types, + LongSupplier longsFn, + DoubleSupplier doublesFn, + BooleanSupplier nullsFn, + Supplier stringFn + ) + { + SettableVectorInputBinding vectorBinding = new SettableVectorInputBinding(vectorSize); + SettableObjectBinding[] objectBindings = new SettableObjectBinding[vectorSize]; + + final boolean hasNulls = NullHandling.sqlCompatible(); + for (Map.Entry entry : types.entrySet()) { + boolean[] nulls = new boolean[vectorSize]; + + switch (entry.getValue()) { + case LONG: + long[] longs = new long[vectorSize]; + for (int i = 0; i < vectorSize; i++) { + nulls[i] = hasNulls && nullsFn.getAsBoolean(); + longs[i] = nulls[i] ? 0L : longsFn.getAsLong(); + if (objectBindings[i] == null) { + objectBindings[i] = new SettableObjectBinding(); + } + objectBindings[i].withBinding(entry.getKey(), nulls[i] ? null : longs[i]); + } + if (hasNulls) { + vectorBinding.addLong(entry.getKey(), longs, nulls); + } else { + vectorBinding.addLong(entry.getKey(), longs); + } + break; + case DOUBLE: + double[] doubles = new double[vectorSize]; + for (int i = 0; i < vectorSize; i++) { + nulls[i] = hasNulls && nullsFn.getAsBoolean(); + doubles[i] = nulls[i] ? 0.0 : doublesFn.getAsDouble(); + if (objectBindings[i] == null) { + objectBindings[i] = new SettableObjectBinding(); + } + objectBindings[i].withBinding(entry.getKey(), nulls[i] ? null : doubles[i]); + } + vectorBinding.addDouble(entry.getKey(), doubles); + break; + case STRING: + String[] strings = new String[vectorSize]; + for (int i = 0; i < vectorSize; i++) { + nulls[i] = hasNulls && nullsFn.getAsBoolean(); + strings[i] = nulls[i] ? null : String.valueOf(stringFn.get()); + if (objectBindings[i] == null) { + objectBindings[i] = new SettableObjectBinding(); + } + objectBindings[i].withBinding(entry.getKey(), nulls[i] ? null : strings[i]); + } + vectorBinding.addString(entry.getKey(), strings); + break; + } + } + + return new NonnullPair<>(objectBindings, vectorBinding); + } + + static class SettableObjectBinding implements Expr.ObjectBinding + { + private final Map bindings; + + SettableObjectBinding() + { + this.bindings = new HashMap<>(); + } + + @Nullable + @Override + public Object get(String name) + { + return bindings.get(name); + } + + public SettableObjectBinding withBinding(String name, @Nullable Object value) + { + bindings.put(name, value); + return this; + } + } + + static class SettableVectorInputBinding implements Expr.VectorInputBinding + { + private final Map nulls; + private final Map longs; + private final Map doubles; + private final Map objects; + private final Map types; + + private final int vectorSize; + + SettableVectorInputBinding(int vectorSize) + { + this.nulls = new HashMap<>(); + this.longs = new HashMap<>(); + this.doubles = new HashMap<>(); + this.objects = new HashMap<>(); + this.types = new HashMap<>(); + this.vectorSize = vectorSize; + } + + public SettableVectorInputBinding addBinding(String name, ExprType type, boolean[] nulls) + { + this.nulls.put(name, nulls); + this.types.put(name, type); + return this; + } + public SettableVectorInputBinding addLong(String name, long[] longs) + { + return addLong(name, longs, new boolean[longs.length]); + } + + public SettableVectorInputBinding addLong(String name, long[] longs, boolean[] nulls) + { + assert longs.length == vectorSize; + this.longs.put(name, longs); + return addBinding(name, ExprType.LONG, nulls); + } + + public SettableVectorInputBinding addDouble(String name, double[] doubles) + { + return addDouble(name, doubles, new boolean[doubles.length]); + } + + public SettableVectorInputBinding addDouble(String name, double[] doubles, boolean[] nulls) + { + assert doubles.length == vectorSize; + this.doubles.put(name, doubles); + return addBinding(name, ExprType.DOUBLE, nulls); + } + + public SettableVectorInputBinding addString(String name, String[] strings) + { + assert strings.length == vectorSize; + this.objects.put(name, strings); + return addBinding(name, ExprType.STRING, new boolean[strings.length]); + } + + @Override + public T[] getObjectVector(String name) + { + return (T[]) objects.get(name); + } + + @Override + public ExprType getType(String name) + { + return types.get(name); + } + + @Override + public long[] getLongVector(String name) + { + return longs.get(name); + } + + @Override + public double[] getDoubleVector(String name) + { + return doubles.get(name); + } + + @Override + public boolean[] getNullVector(String name) + { + return nulls.get(name); + } + + @Override + public int getMaxVectorSize() + { + return vectorSize; + } + + @Override + public int getCurrentVectorSize() + { + return vectorSize; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index 8f539b0e9ed7..8e191ea03051 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -19,6 +19,7 @@ package org.apache.druid.query.aggregation; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.Pair; @@ -30,6 +31,8 @@ import org.apache.druid.segment.DoubleColumnSelector; import org.apache.druid.segment.FloatColumnSelector; import org.apache.druid.segment.LongColumnSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; @@ -191,7 +194,7 @@ static ColumnValueSelector makeColumnValueSelectorWithFloatDefault( ) { if ((fieldName == null) == (fieldExpression == null)) { - throw new IllegalArgumentException("Only one of fieldName and fieldExpression should be non-null"); + throw new IllegalArgumentException("Only one of fieldName or expression should be non-null"); } if (fieldName != null) { return metricFactory.makeColumnValueSelector(fieldName); @@ -225,6 +228,22 @@ public boolean isNull() } } + public static VectorValueSelector makeVectorValueSelector( + VectorColumnSelectorFactory columnSelectorFactory, + String fieldName, + String expression, + Supplier fieldExpression + ) + { + if ((fieldName == null) == (expression == null)) { + throw new IllegalArgumentException("Only one of fieldName or expression should be non-null"); + } + if (expression != null) { + return ExpressionSelectors.makeVectorValueSelector(columnSelectorFactory, fieldExpression.get()); + } + return columnSelectorFactory.makeValueSelector(fieldName); + } + /** * Only one of fieldName and fieldExpression should be non-null */ diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 46f017b24e5f..ce22009f2577 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseDoubleColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseDoubleColumnValueSelector s return new DoubleMaxBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,16 +80,6 @@ protected VectorAggregator factorizeVector( return new DoubleMaxVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregatorFactory.java index f0f4cf6ae4a6..9c068ce6404b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseDoubleColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseDoubleColumnValueSelector s return new DoubleMinBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,16 +80,6 @@ protected VectorAggregator factorizeVector( return new DoubleMinVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregatorFactory.java index 0e5867cf6f00..1753d18103cf 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -25,9 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseDoubleColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; -import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -74,23 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseDoubleColumnValueSelector s return new DoubleSumBufferAggregator(selector); } - - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || ValueType.isNumeric(capabilities.getType())); - } - return expression == null; - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FloatMaxAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FloatMaxAggregatorFactory.java index 328f633ba523..9398f622250a 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FloatMaxAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FloatMaxAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseFloatColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseFloatColumnValueSelector se return new FloatMaxBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,17 +80,6 @@ protected VectorAggregator factorizeVector( return new FloatMaxVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FloatMinAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FloatMinAggregatorFactory.java index 27059c461f85..465aa381d41e 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FloatMinAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FloatMinAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseFloatColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseFloatColumnValueSelector se return new FloatMinBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,16 +80,6 @@ protected VectorAggregator factorizeVector( return new FloatMinVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/FloatSumAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/FloatSumAggregatorFactory.java index 0c61920fc28d..3175b401ad29 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/FloatSumAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/FloatSumAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseFloatColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,22 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseFloatColumnValueSelector se return new FloatSumBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/LongMaxAggregatorFactory.java index a193f346314e..a9bc7c5ae57b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseLongColumnValueSelector sel return new LongMaxBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,16 +80,6 @@ protected VectorAggregator factorizeVector( return new LongMaxVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/LongMinAggregatorFactory.java index ffa03741b4ad..18a3ee684849 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/LongMinAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseLongColumnValueSelector sel return new LongMinBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,16 +80,6 @@ protected VectorAggregator factorizeVector( return new LongMinVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/LongSumAggregatorFactory.java index 337ce1817b63..8c9364f0de59 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/LongSumAggregatorFactory.java @@ -25,8 +25,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.ColumnInspector; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; @@ -73,12 +71,6 @@ protected BufferAggregator buildBufferAggregator(BaseLongColumnValueSelector sel return new LongSumBufferAggregator(selector); } - @Override - protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) - { - return columnSelectorFactory.makeValueSelector(fieldName); - } - @Override protected VectorAggregator factorizeVector( VectorColumnSelectorFactory columnSelectorFactory, @@ -88,16 +80,6 @@ protected VectorAggregator factorizeVector( return new LongSumVectorAggregator(selector); } - @Override - public boolean canVectorize(ColumnInspector columnInspector) - { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return expression == null && (capabilities == null || capabilities.getType().isNumeric()); - } - return expression == null; - } - @Override @Nullable public Object combine(@Nullable Object lhs, @Nullable Object rhs) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 6fdeee3c1f23..c9b0d5f02918 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -28,11 +28,15 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.Parser; import org.apache.druid.segment.BaseDoubleColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; import java.util.Collections; @@ -119,6 +123,12 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) ); } + @Override + protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) + { + return AggregatorUtil.makeVectorValueSelector(columnSelectorFactory, fieldName, expression, fieldExpression); + } + private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory) { if (fieldName != null) { @@ -235,6 +245,19 @@ public String getExpression() return expression; } + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + if (fieldName != null) { + final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); + return capabilities == null || ValueType.isNumeric(capabilities.getType()); + } + if (expression != null) { + return fieldExpression.get().canVectorize(ExpressionSelectors.makeInspectorBindingTypes(columnInspector)); + } + return false; + } + protected abstract double nullValue(); protected abstract Aggregator buildAggregator(BaseDoubleColumnValueSelector selector); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java index ba008065193a..53c50b3714b5 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -28,10 +28,14 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.Parser; import org.apache.druid.segment.BaseFloatColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; import java.util.Collections; @@ -110,6 +114,12 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) ); } + @Override + protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) + { + return AggregatorUtil.makeVectorValueSelector(columnSelectorFactory, fieldName, expression, fieldExpression); + } + @Override public Object deserialize(Object object) { @@ -214,6 +224,19 @@ public String getExpression() return expression; } + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + if (fieldName != null) { + final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); + return capabilities == null || ValueType.isNumeric(capabilities.getType()); + } + if (expression != null) { + return fieldExpression.get().canVectorize(ExpressionSelectors.makeInspectorBindingTypes(columnInspector)); + } + return false; + } + private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory) { if (fieldName != null) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java index 6263441169e9..859875ac81dc 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java @@ -28,10 +28,14 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.Parser; import org.apache.druid.segment.BaseLongColumnValueSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; import java.util.Collections; @@ -116,6 +120,12 @@ protected ColumnValueSelector selector(ColumnSelectorFactory metricFactory) ); } + @Override + protected VectorValueSelector vectorSelector(VectorColumnSelectorFactory columnSelectorFactory) + { + return AggregatorUtil.makeVectorValueSelector(columnSelectorFactory, fieldName, expression, fieldExpression); + } + @Override public Object deserialize(Object object) { @@ -217,6 +227,19 @@ public String getExpression() return expression; } + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + if (fieldName != null) { + final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); + return capabilities == null || ValueType.isNumeric(capabilities.getType()); + } + if (expression != null) { + return fieldExpression.get().canVectorize(ExpressionSelectors.makeInspectorBindingTypes(columnInspector)); + } + return false; + } + private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory) { if (fieldName != null) { diff --git a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java index a3a95306c0ae..d5bd8eea72d9 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java @@ -26,6 +26,9 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.LongUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -120,6 +123,31 @@ public ExprType getOutputType(InputBindingTypes inputTypes) return ExprType.LONG; } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return args.get(0).canVectorize(inputTypes); + } + + @Override + public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + { + VectorExprProcessor processor; + processor = new LongUnivariateFunctionVectorProcessor( + CastToTypeVectorProcessor.castToType(args.get(0).buildVectorized(inputTypes), ExprType.LONG), + inputTypes.getMaxVectorSize() + ) + { + @Override + public long apply(long input) + { + return granularity.bucketStart(DateTimes.utc(input)).getMillis(); + } + }; + + return (VectorExprProcessor) processor; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java index 4110dbdfcece..1f9b1dac6b92 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexColumnSelectorFactory.java @@ -189,7 +189,10 @@ private T getCachedColumn(final String columnName, final public ColumnCapabilities getColumnCapabilities(String columnName) { if (virtualColumns.exists(columnName)) { - return virtualColumns.getColumnCapabilities(columnName); + return virtualColumns.getColumnCapabilities( + baseColumnName -> QueryableIndexStorageAdapter.getColumnCapabilities(index, baseColumnName), + columnName + ); } return QueryableIndexStorageAdapter.getColumnCapabilities(index, columnName); diff --git a/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java index f7299dbad01b..c4b86518856e 100644 --- a/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java @@ -246,6 +246,12 @@ default VectorObjectSelector makeVectorObjectSelector( */ ColumnCapabilities capabilities(String columnName); + + default ColumnCapabilities capabilities(ColumnInspector inspector, String columnName) + { + return capabilities(columnName); + } + /** * Returns a list of columns that this virtual column will access. This may include the * names of other virtual columns. May be empty if a virtual column doesn't access any diff --git a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java index db57bcd8ee72..22d4a5804d90 100644 --- a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java @@ -386,13 +386,28 @@ public ColumnCapabilities getColumnCapabilities(String columnName) } @Nullable - public ColumnCapabilities getColumnCapabilitiesWithFallback(StorageAdapter adapter, String columnName) + public ColumnCapabilities getColumnCapabilities(ColumnInspector inspector, String columnName) { - final ColumnCapabilities virtualColumnCapabilities = getColumnCapabilities(columnName); + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn != null) { + return Preconditions.checkNotNull( + virtualColumn.capabilities(inspector, columnName), + "capabilities for column[%s]", + columnName + ); + } else { + return null; + } + } + + @Nullable + public ColumnCapabilities getColumnCapabilitiesWithFallback(ColumnInspector inspector, String columnName) + { + final ColumnCapabilities virtualColumnCapabilities = getColumnCapabilities(inspector, columnName); if (virtualColumnCapabilities != null) { return virtualColumnCapabilities; } else { - return adapter.getColumnCapabilities(columnName); + return inspector.getColumnCapabilities(columnName); } } diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java b/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java index ba77d92f15ba..5247f9186412 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java @@ -40,6 +40,7 @@ import org.apache.druid.segment.vector.MultiValueDimensionVectorSelector; import org.apache.druid.segment.vector.ReadableVectorOffset; import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; +import org.apache.druid.segment.vector.VectorObjectSelector; import javax.annotation.Nullable; import java.io.IOException; @@ -487,6 +488,56 @@ public int getMaxVectorSize() return new QueryableMultiValueDimensionVectorSelector(); } + @Override + public VectorObjectSelector makeVectorObjectSelector(ReadableVectorOffset offset) + { + if (!hasMultipleValues()) { + class DictionaryEncodedStringSingleValueVectorObjectSelector implements VectorObjectSelector + { + private final int[] vector = new int[offset.getMaxVectorSize()]; + private final String[] strings = new String[offset.getMaxVectorSize()]; + private int id = ReadableVectorOffset.NULL_ID; + + @Override + + public Object[] getObjectVector() + { + if (id == offset.getId()) { + return strings; + } + + if (offset.isContiguous()) { + column.get(vector, offset.getStartOffset(), offset.getCurrentVectorSize()); + } else { + column.get(vector, offset.getOffsets(), offset.getCurrentVectorSize()); + } + for (int i = 0; i < offset.getCurrentVectorSize(); i++) { + strings[i] = lookupName(vector[i]); + } + id = offset.getId(); + + return strings; + } + + @Override + public int getMaxVectorSize() + { + return offset.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return offset.getCurrentVectorSize(); + } + } + + return new DictionaryEncodedStringSingleValueVectorObjectSelector(); + } else { + throw new UnsupportedOperationException("Multivalue string object selector not implemented yet"); + } + } + @Override public void close() throws IOException { diff --git a/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java b/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java index a7683963e68c..71252506c876 100644 --- a/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java +++ b/processing/src/main/java/org/apache/druid/segment/generator/GeneratorBasicSchemas.java @@ -311,6 +311,52 @@ public class GeneratorBasicSchemas SCHEMA_INFO_BUILDER.put("nulls-and-non-nulls", nullsSchema); } + static { + // schema for benchmarking expressions + List expressionsTestsSchemaColumns = ImmutableList.of( + // string dims + GeneratorColumnSchema.makeSequential("string1", ValueType.STRING, false, 1, null, 0, 10000), + GeneratorColumnSchema.makeLazyZipf("string2", ValueType.STRING, false, 1, null, 1, 100, 1.5), + GeneratorColumnSchema.makeLazyZipf("string3", ValueType.STRING, false, 1, 0.1, 1, 1_000_000, 2.0), + GeneratorColumnSchema.makeLazyDiscreteUniform("string4", ValueType.STRING, false, 1, null, 1, 10_000), + GeneratorColumnSchema.makeLazyDiscreteUniform("string5", ValueType.STRING, false, 1, 0.3, 1, 1_000_000), + + // numeric dims + GeneratorColumnSchema.makeSequential("long1", ValueType.LONG, false, 1, null, 0, 10000), + GeneratorColumnSchema.makeLazyZipf("long2", ValueType.LONG, false, 1, null, 1, 101, 1.5), + GeneratorColumnSchema.makeLazyZipf("long3", ValueType.LONG, false, 1, 0.1, -1_000_000, 1_000_000, 2.0), + GeneratorColumnSchema.makeLazyDiscreteUniform("long4", ValueType.LONG, false, 1, null, -10_000, 10000), + GeneratorColumnSchema.makeLazyDiscreteUniform("long5", ValueType.LONG, false, 1, 0.3, -1_000_000, 1_000_000), + + GeneratorColumnSchema.makeLazyZipf("double1", ValueType.DOUBLE, false, 1, null, 1, 101, 1.5), + GeneratorColumnSchema.makeLazyZipf("double2", ValueType.DOUBLE, false, 1, 0.1, -1_000_000, 1_000_000, 2.0), + GeneratorColumnSchema.makeContinuousUniform("double3", ValueType.DOUBLE, false, 1, null, -9000.0, 9000.0), + GeneratorColumnSchema.makeContinuousUniform("double4", ValueType.DOUBLE, false, 1, null, -1_000_000, 1_000_000), + GeneratorColumnSchema.makeLazyZipf("double5", ValueType.DOUBLE, false, 1, 0.1, 0, 1000, 2.0), + + GeneratorColumnSchema.makeLazyZipf("float1", ValueType.FLOAT, false, 1, null, 1, 101, 1.5), + GeneratorColumnSchema.makeLazyZipf("float2", ValueType.FLOAT, false, 1, 0.1, -1_000_000, 1_000_000, 2.0), + GeneratorColumnSchema.makeContinuousUniform("float3", ValueType.FLOAT, false, 1, null, -9000.0, 9000.0), + GeneratorColumnSchema.makeContinuousUniform("float4", ValueType.FLOAT, false, 1, null, -1_000_000, 1_000_000), + GeneratorColumnSchema.makeLazyZipf("float5", ValueType.FLOAT, false, 1, 0.1, 0, 1000, 2.0) + + ); + + List aggs = new ArrayList<>(); + aggs.add(new CountAggregatorFactory("rows")); + + Interval interval = Intervals.of("2000-01-01/P1D"); + + GeneratorSchemaInfo expressionsTestsSchema = new GeneratorSchemaInfo( + expressionsTestsSchemaColumns, + aggs, + interval, + false + ); + + SCHEMA_INFO_BUILDER.put("expression-testbench", expressionsTestsSchema); + } + static { // simple 'wide' schema with null valued rows, high cardinality columns, no aggs on numeric columns // essentially 'nulls-and-non-nulls' with a ton of extra zipf columns of each type with a variety of value diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java index 4773871a3e11..6ecb83f47a9e 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexColumnSelectorFactory.java @@ -127,7 +127,7 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) public ColumnCapabilities getColumnCapabilities(String columnName) { if (virtualColumns.exists(columnName)) { - return virtualColumns.getColumnCapabilities(columnName); + return virtualColumns.getColumnCapabilities(adapter, columnName); } // Use adapter.getColumnCapabilities instead of index.getCapabilities (see note in IncrementalIndexStorageAdapater) diff --git a/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java index a92f0fad91e9..8b5e91763331 100644 --- a/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/vector/QueryableIndexVectorColumnSelectorFactory.java @@ -266,7 +266,10 @@ private BaseColumn getCachedColumn(final String columnName) public ColumnCapabilities getColumnCapabilities(final String columnName) { if (virtualColumns.exists(columnName)) { - return virtualColumns.getColumnCapabilities(columnName); + return virtualColumns.getColumnCapabilities( + baseColumnName -> QueryableIndexStorageAdapter.getColumnCapabilities(index, baseColumnName), + columnName + ); } return QueryableIndexStorageAdapter.getColumnCapabilities(index, columnName); } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java index f2a0571d610a..8ac8e19ad327 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java @@ -28,13 +28,16 @@ import org.apache.druid.java.util.common.UOE; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.Parser; +import org.apache.druid.math.expr.vector.VectorExprProcessor; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.expression.ExprUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.BaseSingleValueDimensionSelector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.ConstantExprEvalSelector; @@ -44,6 +47,11 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.data.IndexedInts; +import org.apache.druid.segment.vector.NilVectorSelector; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorSizeInspector; +import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.util.Arrays; @@ -126,6 +134,32 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) }; } + + + public static VectorValueSelector makeVectorValueSelector( + VectorColumnSelectorFactory factory, + Expr expression + ) + { + final Expr.BindingAnalysis exprAnalysis = expression.analyzeInputs(); + Parser.validateExpr(expression, exprAnalysis); + final Expr.VectorInputBinding bindings = createVectorBindings(exprAnalysis, factory); + final VectorExprProcessor processor = expression.buildVectorized(bindings); + return new ExpressionVectorValueSelector(processor, bindings); + } + + public static VectorObjectSelector makeVectorObjectSelector( + VectorColumnSelectorFactory factory, + Expr expression + ) + { + final Expr.BindingAnalysis exprAnalysis = expression.analyzeInputs(); + Parser.validateExpr(expression, exprAnalysis); + final Expr.VectorInputBinding bindings = createVectorBindings(exprAnalysis, factory); + final VectorExprProcessor processor = expression.buildVectorized(bindings); + return new ExpressionVectorObjectSelector(processor, bindings); + } + /** * Makes a ColumnValueSelector whose getObject method returns an {@link ExprEval}. * @@ -164,8 +198,7 @@ public static ColumnValueSelector makeExprEvalSelector( } } - final Pair, Set> arrayUsage = - examineColumnSelectorFactoryArrays(columnSelectorFactory, bindingAnalysis, columns); + final Pair, Set> arrayUsage = findArrayInputBindings(columnSelectorFactory, bindingAnalysis); final Set actualArrays = arrayUsage.lhs; final Set unknownIfArrays = arrayUsage.rhs; @@ -235,8 +268,7 @@ && canMapOverDictionary(bindingAnalysis, capabilities.hasMultipleValues()) } } - final Pair, Set> arrayUsage = - examineColumnSelectorFactoryArrays(columnSelectorFactory, bindingAnalysis, columns); + final Pair, Set> arrayUsage = findArrayInputBindings(columnSelectorFactory, bindingAnalysis); final Set actualArrays = arrayUsage.lhs; final Set unknownIfArrays = arrayUsage.rhs; @@ -337,6 +369,7 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) } } + /** * Returns whether an expression can be applied to unique values of a particular column (like those in a dictionary) * rather than being applied to each row individually. @@ -369,8 +402,7 @@ private static Expr.ObjectBinding createBindings( final Map> suppliers = new HashMap<>(); final List columns = bindingAnalysis.getRequiredBindingsList(); for (String columnName : columns) { - final ColumnCapabilities columnCapabilities = columnSelectorFactory - .getColumnCapabilities(columnName); + final ColumnCapabilities columnCapabilities = columnSelectorFactory.getColumnCapabilities(columnName); final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; final boolean multiVal = columnCapabilities != null && columnCapabilities.hasMultipleValues().isTrue(); final Supplier supplier; @@ -420,6 +452,33 @@ private static Expr.ObjectBinding createBindings( } } + private static Expr.VectorInputBinding createVectorBindings( + Expr.BindingAnalysis bindingAnalysis, + VectorColumnSelectorFactory vectorColumnSelectorFactory + ) + { + VectorSelectorVectorInputBinding binding = new VectorSelectorVectorInputBinding(vectorColumnSelectorFactory.getVectorSizeInspector()); + final List columns = bindingAnalysis.getRequiredBindingsList(); + for (String columnName : columns) { + final ColumnCapabilities columnCapabilities = vectorColumnSelectorFactory.getColumnCapabilities(columnName); + final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; + final boolean multiVal = columnCapabilities != null && columnCapabilities.hasMultipleValues().isTrue(); + + switch (nativeType) { + case FLOAT: + case DOUBLE: + binding.addNumeric(columnName, ExprType.DOUBLE, vectorColumnSelectorFactory.makeValueSelector(columnName)); + break; + case LONG: + binding.addNumeric(columnName, ExprType.LONG, vectorColumnSelectorFactory.makeValueSelector(columnName)); + break; + default: + binding.addObjectSelector(columnName, ExprType.STRING, vectorColumnSelectorFactory.makeObjectSelector(columnName)); + } + } + return binding; + } + /** * Wraps a {@link ColumnValueSelector} and uses it to supply numeric values in a null-aware way. * @@ -597,18 +656,17 @@ public static Object coerceEvalToSelectorObject(ExprEval eval) /** * Returns pair of columns which are definitely multi-valued, or 'actual' arrays, and those which we are unable to - * discern from the {@link ColumnSelectorFactory#getColumnCapabilities(String)}, or 'unknown' arrays. + * discern from the {@link ColumnInspector#getColumnCapabilities(String)}, or 'unknown' arrays. */ - private static Pair, Set> examineColumnSelectorFactoryArrays( - ColumnSelectorFactory columnSelectorFactory, - Expr.BindingAnalysis bindingAnalysis, - List columns + public static Pair, Set> findArrayInputBindings( + ColumnInspector inspector, + Expr.BindingAnalysis bindingAnalysis ) { final Set actualArrays = new HashSet<>(); final Set unknownIfArrays = new HashSet<>(); - for (String column : columns) { - final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(column); + for (String column : bindingAnalysis.getRequiredBindings()) { + final ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); if (capabilities != null) { if (capabilities.hasMultipleValues().isTrue()) { actualArrays.add(column); @@ -626,4 +684,117 @@ private static Pair, Set> examineColumnSelectorFactoryArrays return new Pair<>(actualArrays, unknownIfArrays); } + + public static Expr.InputBindingTypes makeInspectorBindingTypes(ColumnInspector inspector) + { + return new ColumnInspectorVectorInputBindingTypes(inspector); + } + + static class VectorSelectorVectorInputBinding implements Expr.VectorInputBinding + { + private final Map numeric; + private final Map objects; + private final Map types; + private final NilVectorSelector nilSelector; + + private final VectorSizeInspector sizeInspector; + + public VectorSelectorVectorInputBinding(VectorSizeInspector sizeInspector) + { + this.numeric = new HashMap<>(); + this.objects = new HashMap<>(); + this.types = new HashMap<>(); + this.sizeInspector = sizeInspector; + this.nilSelector = NilVectorSelector.create(sizeInspector); + } + + public VectorSelectorVectorInputBinding addNumeric(String name, ExprType type, VectorValueSelector selector) + { + numeric.put(name, selector); + types.put(name, type); + return this; + } + + public VectorSelectorVectorInputBinding addObjectSelector(String name, ExprType type, VectorObjectSelector selector) + { + objects.put(name, selector); + types.put(name, type); + return this; + } + + @Override + public T[] getObjectVector(String name) + { + if (objects.containsKey(name)) { + return (T[]) objects.get(name).getObjectVector(); + } + return (T[]) nilSelector.getObjectVector(); + } + + @Override + public ExprType getType(String name) + { + return types.get(name); + } + + @Override + public long[] getLongVector(String name) + { + if (numeric.containsKey(name)) { + return numeric.get(name).getLongVector(); + } + return nilSelector.getLongVector(); + } + + @Override + public double[] getDoubleVector(String name) + { + if (numeric.containsKey(name)) { + return numeric.get(name).getDoubleVector(); + } + return nilSelector.getDoubleVector(); + } + + @Override + public boolean[] getNullVector(String name) + { + if (numeric.containsKey(name)) { + return numeric.get(name).getNullVector(); + } + return nilSelector.getNullVector(); + } + + @Override + public int getMaxVectorSize() + { + return sizeInspector.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return sizeInspector.getCurrentVectorSize(); + } + } + + static class ColumnInspectorVectorInputBindingTypes implements Expr.InputBindingTypes + { + private final ColumnInspector inspector; + + public ColumnInspectorVectorInputBindingTypes(ColumnInspector inspector) + { + this.inspector = inspector; + } + + @Nullable + @Override + public ExprType getType(String name) + { + ColumnCapabilities capabilities = inspector.getColumnCapabilities(name); + if (capabilities != null) { + return ExprType.fromValueType(capabilities.getType()); + } + return null; + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java new file mode 100644 index 000000000000..b7346fb0dff3 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import com.google.common.base.Preconditions; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.segment.vector.VectorObjectSelector; + +public class ExpressionVectorObjectSelector implements VectorObjectSelector +{ + final Expr.VectorInputBinding bindings; + final VectorExprProcessor processor; + + public ExpressionVectorObjectSelector(VectorExprProcessor processor, Expr.VectorInputBinding bindings) + { + this.processor = Preconditions.checkNotNull(processor, "processor"); + this.bindings = Preconditions.checkNotNull(bindings, "bindings"); + } + + @Override + public Object[] getObjectVector() + { + return processor.evalVector(bindings).getObjectVector(); + } + + @Override + public int getMaxVectorSize() + { + return bindings.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return bindings.getCurrentVectorSize(); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java new file mode 100644 index 000000000000..63bbcf57ebcf --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import com.google.common.base.Preconditions; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; + +public class ExpressionVectorValueSelector implements VectorValueSelector +{ + final Expr.VectorInputBinding bindings; + final VectorExprProcessor processor; + final float[] floats; + + public ExpressionVectorValueSelector(VectorExprProcessor processor, Expr.VectorInputBinding bindings) + { + this.processor = Preconditions.checkNotNull(processor, "processor"); + this.bindings = Preconditions.checkNotNull(bindings, "bindings"); + this.floats = new float[bindings.getMaxVectorSize()]; + } + + @Override + public long[] getLongVector() + { + return processor.evalVector(bindings).getLongVector(); + } + + @Override + public float[] getFloatVector() + { + final double[] doubles = processor.evalVector(bindings).getDoubleVector(); + for (int i = 0; i < bindings.getCurrentVectorSize(); i++) { + floats[i] = (float) doubles[i]; + } + return floats; + } + + @Override + public double[] getDoubleVector() + { + return processor.evalVector(bindings).getDoubleVector(); + } + + @Nullable + @Override + public boolean[] getNullVector() + { + return processor.evalVector(bindings).getNullVector(); + } + + @Override + public int getMaxVectorSize() + { + return bindings.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return bindings.getCurrentVectorSize(); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index 84a478a0de12..43503fa4e350 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -27,11 +27,14 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.Parser; import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DimensionSelector; @@ -39,10 +42,14 @@ import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.util.List; import java.util.Objects; +import java.util.Set; public class ExpressionVirtualColumn implements VirtualColumn { @@ -129,6 +136,35 @@ public ColumnValueSelector makeColumnValueSelector(String columnName, ColumnS return ExpressionSelectors.makeColumnValueSelector(factory, parsedExpression.get()); } + + @Override + public boolean canVectorize(ColumnInspector inspector) + { + Expr expr = parsedExpression.get(); + Expr.BindingAnalysis analysis = expr.analyzeInputs(); + Expr.InputBindingTypes inputTypes = ExpressionSelectors.makeInspectorBindingTypes(inspector); + // we don't currently support multi-value inputs or outputs for vectorized expressions + return !analysis.hasInputArrays() && + !analysis.isOutputArray() && + analysis.getRequiredBindingsList().stream().noneMatch(column -> { + ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); + return capabilities == null || capabilities.hasMultipleValues().isMaybeTrue(); + }) && + parsedExpression.get().canVectorize(inputTypes); + } + + @Override + public VectorValueSelector makeVectorValueSelector(String columnName, VectorColumnSelectorFactory factory) + { + return ExpressionSelectors.makeVectorValueSelector(factory, parsedExpression.get()); + } + + @Override + public VectorObjectSelector makeVectorObjectSelector(String columnName, VectorColumnSelectorFactory factory) + { + return ExpressionSelectors.makeVectorObjectSelector(factory, parsedExpression.get()); + } + @Override public ColumnCapabilities capabilities(String columnName) { @@ -138,6 +174,53 @@ public ColumnCapabilities capabilities(String columnName) return new ColumnCapabilitiesImpl().setType(outputType); } + @Override + public ColumnCapabilities capabilities(ColumnInspector inspector, String columnName) + { + final ExprType outputType = parsedExpression.get().getOutputType( + ExpressionSelectors.makeInspectorBindingTypes(inspector) + ); + + if (outputType != null) { + final ValueType valueType = ExprType.toValueType(outputType); + if (valueType.isNumeric()) { + // numbers are easy + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ExprType.toValueType(outputType)); + } + if (valueType.isArray()) { + // always a multi-value string since wider engine does not yet support array types + return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(ValueType.STRING); + } + + final Expr.BindingAnalysis analysis = parsedExpression.get().analyzeInputs(); + if (analysis.isOutputArray()) { + // always a multi-value string since wider engine does not yet support array types + return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(ValueType.STRING); + } + + // dupe of some logic in ExpressionSelectors to detect if any possiblility of this producing multi value outputs + final Pair, Set> arrayUsage = ExpressionSelectors.findArrayInputBindings(inspector, analysis); + final Set actualArrays = arrayUsage.lhs; + final Set unknownIfArrays = arrayUsage.rhs; + final long needsAppliedCount = + analysis.getRequiredBindings() + .stream() + .filter(c -> actualArrays.contains(c) && !analysis.getArrayBindings().contains(c)) + .count(); + + // unapplied multi-valued inputs result in multi-valued outputs, and unknowns are unknown + if (unknownIfArrays.size() > 0 || needsAppliedCount > 0) { + // always a multi-value string since wider engine does not yet support array types + return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(ValueType.STRING); + } + // if we got here, lets call it single value string output + return new ColumnCapabilitiesImpl().setType(ValueType.STRING) + .setHasMultipleValues(false) + .setDictionaryEncoded(false); + } + return capabilities(columnName); + } + @Override public List requiredColumns() { diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/VirtualizedColumnSelectorFactory.java b/processing/src/main/java/org/apache/druid/segment/virtual/VirtualizedColumnSelectorFactory.java index 35805f5b5cab..d37de2ea3c6e 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/VirtualizedColumnSelectorFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/VirtualizedColumnSelectorFactory.java @@ -68,7 +68,7 @@ public ColumnValueSelector makeColumnValueSelector(String columnName) public ColumnCapabilities getColumnCapabilities(String columnName) { if (virtualColumns.exists(columnName)) { - return virtualColumns.getColumnCapabilities(columnName); + return virtualColumns.getColumnCapabilities(baseFactory, columnName); } else { return baseFactory.getColumnCapabilities(columnName); } diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java index c89e35b44139..c34e1fd4abbe 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryRunnerTest.java @@ -48,6 +48,7 @@ import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.js.JavaScriptConfig; +import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.BySegmentResultValue; import org.apache.druid.query.BySegmentResultValueClass; import org.apache.druid.query.ChainedExecutionQueryRunner; @@ -3211,9 +3212,6 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy() @Test public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderByUsingMathExpressions() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - final int limit = 14; GroupByQuery.Builder builder = makeQueryBuilder() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) @@ -3364,9 +3362,6 @@ public Sequence run(QueryPlus queryPlus, ResponseContext r @Test public void testGroupByOrderLimit() { - // Cannot vectorize due to expression-based aggregator. - cannotVectorize(); - GroupByQuery.Builder builder = makeQueryBuilder() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) .setInterval("2011-04-02/2011-04-04") @@ -4807,9 +4802,6 @@ public void testSubqueryWithExtractionFnInOuterQuery() @Test public void testDifferentGroupingSubquery() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - GroupByQuery subquery = makeQueryBuilder() .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) @@ -10652,6 +10644,44 @@ public void testGroupByNoMatchingPrefilter() TestHelper.assertExpectedObjects(expectedResults, results, "groupBy"); } + @Test + public void testGroupByOnVirtualColumn() + { + if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) { + expectedException.expect(UnsupportedOperationException.class); + } + + GroupByQuery query = makeQueryBuilder() + .setDataSource(QueryRunnerTestHelper.DATA_SOURCE) + .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD) + .setVirtualColumns( + new ExpressionVirtualColumn( + "v", + "qualityDouble * qualityLong", + ValueType.LONG, + ExprMacroTable.nil() + ) + ) + .setDimensions( + new DefaultDimensionSpec("v", "v", ValueType.LONG) + ) + .setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT) + .setGranularity(QueryRunnerTestHelper.ALL_GRAN) + .setLimit(5) + .build(); + + List expectedResults = Arrays.asList( + makeRow(query, "2011-04-01", "v", 10000000L, "rows", 2L), + makeRow(query, "2011-04-01", "v", 12100000L, "rows", 2L), + makeRow(query, "2011-04-01", "v", 14400000L, "rows", 2L), + makeRow(query, "2011-04-01", "v", 16900000L, "rows", 2L), + makeRow(query, "2011-04-01", "v", 19600000L, "rows", 6L) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, "groupBy"); + } + private static ResultRow makeRow(final GroupByQuery query, final String timestamp, final Object... vals) { return GroupByQueryRunnerTestHelper.createExpectedRow(query, timestamp, vals); diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java index c67dfb594ce4..ae5aa1e10fa1 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -670,9 +670,6 @@ public void testTimeseriesIntervalOutOfRanges() @Test public void testTimeseriesWithVirtualColumn() { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() .dataSource(QueryRunnerTestHelper.DATA_SOURCE) .granularity(QueryRunnerTestHelper.DAY_GRAN) diff --git a/processing/src/test/java/org/apache/druid/segment/generator/DataGeneratorTest.java b/processing/src/test/java/org/apache/druid/segment/generator/DataGeneratorTest.java index 4f65d377d4ee..814e8ae1bb17 100644 --- a/processing/src/test/java/org/apache/druid/segment/generator/DataGeneratorTest.java +++ b/processing/src/test/java/org/apache/druid/segment/generator/DataGeneratorTest.java @@ -411,6 +411,74 @@ public void testRealRoundingDistributionZeroGetters() Assert.assertEquals(0, dist.getNumericalVariance(), 0); } + @Test + public void testLazyZipf() + { + List schemas = new ArrayList<>(); + RowValueTracker tracker = new RowValueTracker(); + + schemas.add( + GeneratorColumnSchema.makeLazyZipf( + "dimA", + ValueType.STRING, + false, + 1, + null, + 0, + 1220000, + 1.0 + ) + ); + + schemas.add( + GeneratorColumnSchema.makeLazyZipf( + "dimB", + ValueType.FLOAT, + false, + 1, + null, + 99990, + 99999, + 1.0 + ) + ); + + schemas.add( + GeneratorColumnSchema.makeLazyZipf( + "dimC", + ValueType.DOUBLE, + false, + 1, + null, + 0, + 100000, + 1.5 + ) + ); + schemas.add( + GeneratorColumnSchema.makeLazyZipf( + "dimD", + ValueType.LONG, + false, + 1, + null, + 0, + 100000, + 1.5 + ) + ); + + DataGenerator dataGenerator = new DataGenerator(schemas, 9999, 0, 0, 1000.0); + for (int i = 0; i < 100000; i++) { + InputRow row = dataGenerator.nextRow(); + System.out.println("Z-ROW: " + row); + + tracker.addRow(row); + } + + tracker.printStuff(); + } + private static class RowValueTracker { private Map> dimensionMap; diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index c70f7a4dec96..3c6072ad0b15 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -231,6 +231,14 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat final Closer closer, final boolean useParallelMergePoolConfigured ) + { + return createQueryRunnerFactoryConglomerate(closer, getProcessingConfig(useParallelMergePoolConfigured)); + } + + public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( + final Closer closer, + final DruidProcessingConfig processingConfig + ) { final CloseableStupidPool stupidPool = new CloseableStupidPool<>( "TopNQueryRunnerFactory-bufferPool", @@ -250,7 +258,7 @@ public String getDefaultStrategy() return GroupByStrategySelector.STRATEGY_V2; } }, - getProcessingConfig(useParallelMergePoolConfigured) + processingConfig ); final GroupByQueryRunnerFactory groupByQueryRunnerFactory = factoryCloserPair.lhs; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index d1e9c7001906..5cf04895247a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -5912,9 +5912,6 @@ public void testExpressionFilteringAndGrouping() throws Exception @Test public void testExpressionFilteringAndGroupingUsingCastToLong() throws Exception { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " CAST(m1 AS BIGINT) / 2 * 2,\n" @@ -6653,9 +6650,6 @@ public void testCountStarWithTimeFilterOnLongColumnUsingTimestampToMillis() thro @Test public void testSumOfString() throws Exception { - // Cannot vectorize due to expressions in aggregators. - cannotVectorize(); - testQuery( "SELECT SUM(CAST(dim1 AS INTEGER)) FROM druid.foo", ImmutableList.of( @@ -6713,9 +6707,6 @@ public void testSumOfExtractionFn() throws Exception @Test public void testTimeseriesWithTimeFilterOnLongColumnUsingMillisToTimestamp() throws Exception { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT\n" + " FLOOR(MILLIS_TO_TIMESTAMP(cnt) TO YEAR),\n" @@ -9191,9 +9182,6 @@ public void testGroupByFloorWithOrderBy() throws Exception @Test public void testGroupByFloorTimeAndOneOtherDimensionWithOrderBy() throws Exception { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT floor(__time TO year), dim2, COUNT(*)" + " FROM druid.foo" @@ -11432,9 +11420,6 @@ public void testTimeseriesUsingTimeFloorWithTimeShift() throws Exception @Test public void testTimeseriesUsingTimeFloorWithTimestampAdd() throws Exception { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT SUM(cnt), gran FROM (\n" + " SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'P1M') AS gran,\n" @@ -11944,11 +11929,7 @@ public void testTimeseriesWithLimit() throws Exception @Test public void testTimeseriesWithLimitAndOffset() throws Exception { - // Cannot vectorize due to expressions. - cannotVectorize(); - // Timeseries cannot handle offsets, so the query morphs into a groupBy. - testQuery( "SELECT gran, SUM(cnt)\n" + "FROM (\n" @@ -12013,9 +11994,6 @@ public void testTimeseriesWithOrderByAndLimit() throws Exception @Test public void testGroupByTimeAndOtherDimension() throws Exception { - // Cannot vectorize due to virtual columns. - cannotVectorize(); - testQuery( "SELECT dim2, gran, SUM(cnt)\n" + "FROM (SELECT FLOOR(__time TO MONTH) AS gran, dim2, cnt FROM druid.foo) AS x\n" @@ -15939,8 +15917,6 @@ public void testJoinOnConstantShouldFail(Map queryContext) throw @Test public void testRepeatedIdenticalVirtualExpressionGrouping() throws Exception { - cannotVectorize(); - final String query = "SELECT \n" + "\tCASE dim1 WHEN NULL THEN FALSE ELSE TRUE END AS col_a,\n" + "\tCASE dim2 WHEN NULL THEN FALSE ELSE TRUE END AS col_b\n" From 44f9a6cdbdd02e9492431fa69d761a1108216760 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 16 Sep 2020 12:32:47 -0700 Subject: [PATCH 02/17] cleanup --- .../ExpressionVectorSelectorBenchmark.java | 5 +- .../query/SqlExpressionBenchmark.java | 3 +- .../java/org/apache/druid/math/expr/Expr.java | 1 + .../org/apache/druid/math/expr/ExprEval.java | 3 + .../org/apache/druid/math/expr/Function.java | 46 +++++++++++++-- .../expr/vector/StringVectorExprEval.java | 40 ++++++------- .../druid/math/expr/VectorExprSanityTest.java | 1 + .../SimpleDoubleAggregatorFactory.java | 3 +- .../SimpleFloatAggregatorFactory.java | 3 +- .../SimpleLongAggregatorFactory.java | 3 +- .../apache/druid/segment/ColumnInspector.java | 15 ++++- .../apache/druid/segment/VirtualColumns.java | 15 ----- .../segment/virtual/ExpressionSelectors.java | 58 ++++--------------- .../virtual/ExpressionVirtualColumn.java | 7 +-- 14 files changed, 98 insertions(+), 105 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java index 8e83e6f5e83d..a83c686687e9 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java @@ -43,7 +43,6 @@ import org.apache.druid.segment.vector.VectorCursor; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionSelectors; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -131,7 +130,7 @@ public void setup() Expr parsed = Parser.parse(expression, ExprMacroTable.nil()); outputType = parsed.getOutputType( - ExpressionSelectors.makeInspectorBindingTypes(new ColumnInspector() + new ColumnInspector() { @Nullable @Override @@ -139,7 +138,7 @@ public ColumnCapabilities getColumnCapabilities(String column) { return QueryableIndexStorageAdapter.getColumnCapabilities(index, column); } - }) + } ); checkSanity(); } diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 328e44680760..0a098783688c 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -86,7 +86,8 @@ public class SqlExpressionBenchmark NullHandling.initializeForTests(); Calcites.setSystemProperties(); } - static DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() + + private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() { @Override public int intermediateComputeSizeBytes() diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index bfa4e545e367..06c6f047eaef 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -232,6 +232,7 @@ interface VectorInputBinding extends VectorInputBindingTypes long[] getLongVector(String name); double[] getDoubleVector(String name); + @Nullable boolean[] getNullVector(String name); int getCurrentVectorSize(); diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java index 8caa33db1861..52e57309670b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -124,6 +124,9 @@ public static ExprEval bestEffortOf(@Nullable Object val) @Nullable public static Number computeNumber(@Nullable String value) { + if (value == null) { + return null; + } Number rv; Long v = GuavaUtils.tryParseLong(value); // Do NOT use ternary operator here, because it makes Java to convert Long to Double diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 085d02cfa941..2cbb3a1e2a61 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -720,6 +720,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -805,6 +810,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -860,6 +870,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -915,6 +930,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -1229,6 +1249,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -1284,6 +1309,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -1354,6 +1384,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -1409,6 +1444,11 @@ public double apply(double input) } }; } + + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; } } @@ -1839,13 +1879,11 @@ public ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args @Override public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) { - return args.get(0).canVectorize(inputTypes) && (args.get(1).isLiteral() || args.get(1).canVectorize(inputTypes)); + return args.get(0).canVectorize(inputTypes) && args.get(1).isLiteral(); } @Override - public VectorExprProcessor asVectorProcessor( - Expr.VectorInputBindingTypes inputTypes, List args - ) + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return CastToTypeVectorProcessor.castToType( args.get(0).buildVectorized(inputTypes), diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java index 19b80c3788b0..94f5ee0d1bc9 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java @@ -43,19 +43,21 @@ public StringVectorExprEval(String[] values) private void computeNumbers() { - longs = new long[values.length]; - doubles = new double[values.length]; - numericNulls = new boolean[values.length]; - for (int i = 0; i < values.length; i++) { - Number n = ExprEval.computeNumber(values[i]); - if (n != null) { - longs[i] = n.longValue(); - doubles[i] = n.doubleValue(); - numericNulls[i] = false; - } else { - longs[i] = 0L; - doubles[i] = 0.0; - numericNulls[i] = NullHandling.sqlCompatible(); + if (longs == null) { + longs = new long[values.length]; + doubles = new double[values.length]; + numericNulls = new boolean[values.length]; + for (int i = 0; i < values.length; i++) { + Number n = ExprEval.computeNumber(values[i]); + if (n != null) { + longs[i] = n.longValue(); + doubles[i] = n.doubleValue(); + numericNulls[i] = false; + } else { + longs[i] = 0L; + doubles[i] = 0.0; + numericNulls[i] = NullHandling.sqlCompatible(); + } } } } @@ -64,9 +66,7 @@ private void computeNumbers() @Override public boolean[] getNullVector() { - if (numericNulls == null) { - computeNumbers(); - } + computeNumbers(); return numericNulls; } @@ -79,18 +79,14 @@ public ExprType getType() @Override public long[] getLongVector() { - if (longs == null) { - computeNumbers(); - } + computeNumbers(); return longs; } @Override public double[] getDoubleVector() { - if (doubles == null) { - computeNumbers(); - } + computeNumbers(); return doubles; } diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index 4335f9e2571e..889c185ed518 100644 --- a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -402,6 +402,7 @@ public double[] getDoubleVector(String name) return doubles.get(name); } + @Nullable @Override public boolean[] getNullVector(String name) { diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index c9b0d5f02918..41c07fd4a4e3 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -36,7 +36,6 @@ import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; import java.util.Collections; @@ -253,7 +252,7 @@ public boolean canVectorize(ColumnInspector columnInspector) return capabilities == null || ValueType.isNumeric(capabilities.getType()); } if (expression != null) { - return fieldExpression.get().canVectorize(ExpressionSelectors.makeInspectorBindingTypes(columnInspector)); + return fieldExpression.get().canVectorize(columnInspector); } return false; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java index 53c50b3714b5..1f41f4ce2213 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -35,7 +35,6 @@ import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; import java.util.Collections; @@ -232,7 +231,7 @@ public boolean canVectorize(ColumnInspector columnInspector) return capabilities == null || ValueType.isNumeric(capabilities.getType()); } if (expression != null) { - return fieldExpression.get().canVectorize(ExpressionSelectors.makeInspectorBindingTypes(columnInspector)); + return fieldExpression.get().canVectorize(columnInspector); } return false; } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java index 859875ac81dc..ecacbf836b68 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java @@ -35,7 +35,6 @@ import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; -import org.apache.druid.segment.virtual.ExpressionSelectors; import javax.annotation.Nullable; import java.util.Collections; @@ -235,7 +234,7 @@ public boolean canVectorize(ColumnInspector columnInspector) return capabilities == null || ValueType.isNumeric(capabilities.getType()); } if (expression != null) { - return fieldExpression.get().canVectorize(ExpressionSelectors.makeInspectorBindingTypes(columnInspector)); + return fieldExpression.get().canVectorize(columnInspector); } return false; } diff --git a/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java b/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java index 3090455833e3..16c7460eb47f 100644 --- a/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java +++ b/processing/src/main/java/org/apache/druid/segment/ColumnInspector.java @@ -19,11 +19,13 @@ package org.apache.druid.segment; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; import org.apache.druid.segment.column.ColumnCapabilities; import javax.annotation.Nullable; -public interface ColumnInspector +public interface ColumnInspector extends Expr.InputBindingTypes { /** * Returns capabilities of a particular column. @@ -34,4 +36,15 @@ public interface ColumnInspector */ @Nullable ColumnCapabilities getColumnCapabilities(String column); + + @Nullable + @Override + default ExprType getType(String name) + { + ColumnCapabilities capabilities = getColumnCapabilities(name); + if (capabilities != null) { + return ExprType.fromValueType(capabilities.getType()); + } + return null; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java index 22d4a5804d90..7419579b56e4 100644 --- a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java @@ -370,21 +370,6 @@ public VectorObjectSelector makeVectorObjectSelector( return virtualColumn.makeVectorObjectSelector(columnName, columnSelector, offset); } - @Nullable - public ColumnCapabilities getColumnCapabilities(String columnName) - { - final VirtualColumn virtualColumn = getVirtualColumn(columnName); - if (virtualColumn != null) { - return Preconditions.checkNotNull( - virtualColumn.capabilities(columnName), - "capabilities for column[%s]", - columnName - ); - } else { - return null; - } - } - @Nullable public ColumnCapabilities getColumnCapabilities(ColumnInspector inspector, String columnName) { diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java index 8ac8e19ad327..48a31cc9d72f 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java @@ -457,12 +457,11 @@ private static Expr.VectorInputBinding createVectorBindings( VectorColumnSelectorFactory vectorColumnSelectorFactory ) { - VectorSelectorVectorInputBinding binding = new VectorSelectorVectorInputBinding(vectorColumnSelectorFactory.getVectorSizeInspector()); + VectorSelectorsVectorInputBinding binding = new VectorSelectorsVectorInputBinding(vectorColumnSelectorFactory.getVectorSizeInspector()); final List columns = bindingAnalysis.getRequiredBindingsList(); for (String columnName : columns) { final ColumnCapabilities columnCapabilities = vectorColumnSelectorFactory.getColumnCapabilities(columnName); final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; - final boolean multiVal = columnCapabilities != null && columnCapabilities.hasMultipleValues().isTrue(); switch (nativeType) { case FLOAT: @@ -685,12 +684,7 @@ public static Pair, Set> findArrayInputBindings( return new Pair<>(actualArrays, unknownIfArrays); } - public static Expr.InputBindingTypes makeInspectorBindingTypes(ColumnInspector inspector) - { - return new ColumnInspectorVectorInputBindingTypes(inspector); - } - - static class VectorSelectorVectorInputBinding implements Expr.VectorInputBinding + static class VectorSelectorsVectorInputBinding implements Expr.VectorInputBinding { private final Map numeric; private final Map objects; @@ -699,7 +693,7 @@ static class VectorSelectorVectorInputBinding implements Expr.VectorInputBinding private final VectorSizeInspector sizeInspector; - public VectorSelectorVectorInputBinding(VectorSizeInspector sizeInspector) + public VectorSelectorsVectorInputBinding(VectorSizeInspector sizeInspector) { this.numeric = new HashMap<>(); this.objects = new HashMap<>(); @@ -708,14 +702,14 @@ public VectorSelectorVectorInputBinding(VectorSizeInspector sizeInspector) this.nilSelector = NilVectorSelector.create(sizeInspector); } - public VectorSelectorVectorInputBinding addNumeric(String name, ExprType type, VectorValueSelector selector) + public VectorSelectorsVectorInputBinding addNumeric(String name, ExprType type, VectorValueSelector selector) { numeric.put(name, selector); types.put(name, type); return this; } - public VectorSelectorVectorInputBinding addObjectSelector(String name, ExprType type, VectorObjectSelector selector) + public VectorSelectorsVectorInputBinding addObjectSelector(String name, ExprType type, VectorObjectSelector selector) { objects.put(name, selector); types.put(name, type); @@ -725,10 +719,7 @@ public VectorSelectorVectorInputBinding addObjectSelector(String name, ExprType @Override public T[] getObjectVector(String name) { - if (objects.containsKey(name)) { - return (T[]) objects.get(name).getObjectVector(); - } - return (T[]) nilSelector.getObjectVector(); + return (T[]) objects.getOrDefault(name, nilSelector).getObjectVector(); } @Override @@ -740,28 +731,20 @@ public ExprType getType(String name) @Override public long[] getLongVector(String name) { - if (numeric.containsKey(name)) { - return numeric.get(name).getLongVector(); - } - return nilSelector.getLongVector(); + return numeric.getOrDefault(name, nilSelector).getLongVector(); } @Override public double[] getDoubleVector(String name) { - if (numeric.containsKey(name)) { - return numeric.get(name).getDoubleVector(); - } - return nilSelector.getDoubleVector(); + return numeric.getOrDefault(name, nilSelector).getDoubleVector(); } + @Nullable @Override public boolean[] getNullVector(String name) { - if (numeric.containsKey(name)) { - return numeric.get(name).getNullVector(); - } - return nilSelector.getNullVector(); + return numeric.getOrDefault(name, nilSelector).getNullVector(); } @Override @@ -776,25 +759,4 @@ public int getCurrentVectorSize() return sizeInspector.getCurrentVectorSize(); } } - - static class ColumnInspectorVectorInputBindingTypes implements Expr.InputBindingTypes - { - private final ColumnInspector inspector; - - public ColumnInspectorVectorInputBindingTypes(ColumnInspector inspector) - { - this.inspector = inspector; - } - - @Nullable - @Override - public ExprType getType(String name) - { - ColumnCapabilities capabilities = inspector.getColumnCapabilities(name); - if (capabilities != null) { - return ExprType.fromValueType(capabilities.getType()); - } - return null; - } - } } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index 43503fa4e350..b729d20cd760 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -142,7 +142,6 @@ public boolean canVectorize(ColumnInspector inspector) { Expr expr = parsedExpression.get(); Expr.BindingAnalysis analysis = expr.analyzeInputs(); - Expr.InputBindingTypes inputTypes = ExpressionSelectors.makeInspectorBindingTypes(inspector); // we don't currently support multi-value inputs or outputs for vectorized expressions return !analysis.hasInputArrays() && !analysis.isOutputArray() && @@ -150,7 +149,7 @@ public boolean canVectorize(ColumnInspector inspector) ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); return capabilities == null || capabilities.hasMultipleValues().isMaybeTrue(); }) && - parsedExpression.get().canVectorize(inputTypes); + parsedExpression.get().canVectorize(inspector); } @Override @@ -177,9 +176,7 @@ public ColumnCapabilities capabilities(String columnName) @Override public ColumnCapabilities capabilities(ColumnInspector inspector, String columnName) { - final ExprType outputType = parsedExpression.get().getOutputType( - ExpressionSelectors.makeInspectorBindingTypes(inspector) - ); + final ExprType outputType = parsedExpression.get().getOutputType(inspector); if (outputType != null) { final ValueType valueType = ExprType.toValueType(outputType); From 00f764fc92c1a101006e29cf43b8ca730b9428d0 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 16 Sep 2020 14:44:18 -0700 Subject: [PATCH 03/17] fixes --- .../org/apache/druid/math/expr/Function.java | 46 ++++++++----------- .../segment/virtual/ExpressionSelectors.java | 27 +++++++---- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 2cbb3a1e2a61..8ea71f788af5 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -548,36 +548,30 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - ExprType parseType = args.get(0).getOutputType(inputTypes); - VectorExprProcessor processor; + VectorExprProcessor processor = null; if (args.size() == 1) { - VectorExprProcessor inputProcessor = args.get(0).buildVectorized(inputTypes); - // if its already a long, just pass it through - if (ExprType.LONG.equals(inputProcessor.getOutputType())) { - processor = inputProcessor; - } else { - // else, always convert the argument to a string, who cares what it is - processor = new StringLongUnivariateFunctionVectorProcessor( - CastToTypeVectorProcessor.castToType(inputProcessor, ExprType.STRING), - inputTypes.getMaxVectorSize() - ) + processor = new StringLongUnivariateFunctionVectorProcessor( + CastToTypeVectorProcessor.castToType(args.get(0).buildVectorized(inputTypes), ExprType.STRING), + inputTypes.getMaxVectorSize() + ) + { + @Override + public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) { - @Override - public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) - { - try { - longs[i] = Long.parseLong(strings[i], 10); - outputNulls[i] = false; - } - catch (NumberFormatException e) { - longs[i] = 0L; - outputNulls[i] = NullHandling.sqlCompatible(); - } + try { + longs[i] = Long.parseLong(strings[i], 10); + outputNulls[i] = false; } - }; - } - } else { + catch (NumberFormatException e) { + longs[i] = 0L; + outputNulls[i] = NullHandling.sqlCompatible(); + } + } + }; + } + + if (processor == null) { // not yet implemented, how did we get here throw Exprs.cannotVectorize(this); } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java index 48a31cc9d72f..03c763186c86 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java @@ -463,16 +463,23 @@ private static Expr.VectorInputBinding createVectorBindings( final ColumnCapabilities columnCapabilities = vectorColumnSelectorFactory.getColumnCapabilities(columnName); final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; - switch (nativeType) { - case FLOAT: - case DOUBLE: - binding.addNumeric(columnName, ExprType.DOUBLE, vectorColumnSelectorFactory.makeValueSelector(columnName)); - break; - case LONG: - binding.addNumeric(columnName, ExprType.LONG, vectorColumnSelectorFactory.makeValueSelector(columnName)); - break; - default: - binding.addObjectSelector(columnName, ExprType.STRING, vectorColumnSelectorFactory.makeObjectSelector(columnName)); + // null capabilities should be backed by a nil vector selector since it means the column effectively doesnt exist + if (nativeType != null) { + switch (nativeType) { + case FLOAT: + case DOUBLE: + binding.addNumeric(columnName, ExprType.DOUBLE, vectorColumnSelectorFactory.makeValueSelector(columnName)); + break; + case LONG: + binding.addNumeric(columnName, ExprType.LONG, vectorColumnSelectorFactory.makeValueSelector(columnName)); + break; + default: + binding.addObjectSelector( + columnName, + ExprType.STRING, + vectorColumnSelectorFactory.makeObjectSelector(columnName) + ); + } } } return binding; From 8dfb16035b8ffa223991d939338e5ce596b849b7 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 16 Sep 2020 15:51:27 -0700 Subject: [PATCH 04/17] preserve float if explicitly specified --- .../virtual/ExpressionVirtualColumn.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index b729d20cd760..1e43f71e9bf7 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -55,6 +55,7 @@ public class ExpressionVirtualColumn implements VirtualColumn { private final String name; private final String expression; + @Nullable private final ValueType outputType; private final Supplier parsedExpression; @@ -68,7 +69,7 @@ public ExpressionVirtualColumn( { this.name = Preconditions.checkNotNull(name, "name"); this.expression = Preconditions.checkNotNull(expression, "expression"); - this.outputType = outputType != null ? outputType : ValueType.FLOAT; + this.outputType = outputType; this.parsedExpression = Suppliers.memoize(() -> Parser.parse(expression, macroTable)); } @@ -85,7 +86,7 @@ public ExpressionVirtualColumn( // Unfortunately this string representation can't be reparsed into the same expression, might be useful // if the expression system supported that this.expression = parsedExpression.toString(); - this.outputType = outputType != null ? outputType : ValueType.FLOAT; + this.outputType = outputType; this.parsedExpression = Suppliers.ofInstance(parsedExpression); } @@ -102,6 +103,7 @@ public String getExpression() return expression; } + @Nullable @JsonProperty public ValueType getOutputType() { @@ -170,19 +172,22 @@ public ColumnCapabilities capabilities(String columnName) // Note: Ideally we would fill out additional information instead of leaving capabilities as 'unknown', e.g. examine // if the expression in question could potentially return multiple values and anything else. However, we don't // currently have a good way of determining this, so fill this out more once we do - return new ColumnCapabilitiesImpl().setType(outputType); + return new ColumnCapabilitiesImpl().setType(outputType == null ? ValueType.FLOAT : outputType); } @Override public ColumnCapabilities capabilities(ColumnInspector inspector, String columnName) { - final ExprType outputType = parsedExpression.get().getOutputType(inspector); + final ExprType inferredOutputType = parsedExpression.get().getOutputType(inspector); - if (outputType != null) { - final ValueType valueType = ExprType.toValueType(outputType); + if (inferredOutputType != null) { + final ValueType valueType = ExprType.toValueType(inferredOutputType); if (valueType.isNumeric()) { - // numbers are easy - return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ExprType.toValueType(outputType)); + // if float was explicitly specified preserve it, because it will currently never be the computed output type + if (ValueType.FLOAT.equals(outputType)) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.FLOAT); + } + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ExprType.toValueType(inferredOutputType)); } if (valueType.isArray()) { // always a multi-value string since wider engine does not yet support array types @@ -233,11 +238,14 @@ public boolean usesDotNotation() @Override public byte[] getCacheKey() { - return new CacheKeyBuilder(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION) + CacheKeyBuilder builder = new CacheKeyBuilder(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION) .appendString(name) - .appendString(expression) - .appendString(outputType.toString()) - .build(); + .appendString(expression); + + if (outputType != null) { + builder.appendString(outputType.toString()); + } + return builder.build(); } @Override From cc0c851aad7effb5f29ae6bdc385053dab6a39ef Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 16 Sep 2020 16:40:50 -0700 Subject: [PATCH 05/17] oops --- .../apache/druid/query/filter/sql/BloomDimFilterSqlTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/sql/BloomDimFilterSqlTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/sql/BloomDimFilterSqlTest.java index b93b46bc382d..f29fbc2b07c5 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/sql/BloomDimFilterSqlTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/filter/sql/BloomDimFilterSqlTest.java @@ -210,9 +210,6 @@ public void testBloomFilterVirtualColumn() throws Exception @Test public void testBloomFilterVirtualColumnNumber() throws Exception { - // Cannot vectorize due to expression virtual columns. - cannotVectorize(); - BloomKFilter filter = new BloomKFilter(1500); filter.addFloat(20.2f); byte[] bytes = BloomFilterSerializersModule.bloomKFilterToBytes(filter); From ec5937d904170d3850642d84a4a9111b42c3219b Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 16 Sep 2020 20:33:48 -0700 Subject: [PATCH 06/17] null handling fixes, more tests --- ...ubleUnivariateFunctionVectorProcessor.java | 4 +- .../expr/vector/DoubleVectorExprEval.java | 8 +- ...ubleUnivariateFunctionVectorProcessor.java | 4 +- ...LongUnivariateFunctionVectorProcessor.java | 4 +- .../math/expr/vector/LongVectorExprEval.java | 8 +- .../druid/math/expr/VectorExprSanityTest.java | 197 +++++++++++------- 6 files changed, 136 insertions(+), 89 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java index b275638ced3e..47c66d3577fa 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java @@ -37,9 +37,9 @@ public ExprType getOutputType() } @Override - final void processIndex(double[] longs, int i) + final void processIndex(double[] input, int i) { - outValues[i] = apply(longs[i]); + outValues[i] = apply(input[i]); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java index ac8d2acc0aea..bc15a477ae7d 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java @@ -60,7 +60,13 @@ public E asObjectVector(ExprType type) { switch (type) { case STRING: - return (E) Arrays.stream(values).mapToObj(String::valueOf).toArray(String[]::new); + String[] s = new String[values.length]; + if (nulls != null) { + for (int i = 0; i < values.length; i++) { + s[i] = nulls[i] ? null : String.valueOf(values[i]); + } + } + return (E) s; default: throw new IAE("Cannot convert %s to %s object vector", getType(), type); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java index e59e54fb31c8..15eb77d95520 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java @@ -37,9 +37,9 @@ public ExprType getOutputType() } @Override - final void processIndex(long[] longs, int i) + final void processIndex(long[] input, int i) { - outValues[i] = apply(longs[i]); + outValues[i] = apply(input[i]); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java index b54a021507d8..16897a7e070c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java @@ -37,9 +37,9 @@ public ExprType getOutputType() } @Override - final void processIndex(long[] longs, int i) + final void processIndex(long[] input, int i) { - outValues[i] = apply(longs[i]); + outValues[i] = apply(input[i]); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java index 8932342d5593..47ab7509af0d 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java @@ -55,7 +55,13 @@ public E asObjectVector(ExprType type) { switch (type) { case STRING: - return (E) Arrays.stream(values).mapToObj(String::valueOf).toArray(String[]::new); + String[] s = new String[values.length]; + if (nulls != null) { + for (int i = 0; i < values.length; i++) { + s[i] = nulls[i] ? null : String.valueOf(values[i]); + } + } + return (E) s; default: throw new IAE("Cannot convert %s to %s object vector", getType(), type); } diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index 889c185ed518..dfd6c613adc8 100644 --- a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -23,12 +23,14 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.vector.VectorExprEval; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; import javax.annotation.Nullable; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ThreadLocalRandom; @@ -45,7 +47,8 @@ */ public class VectorExprSanityTest extends InitializedNullHandlingTest { - private static final int NUM_ITERATIONS = 100; + private static final Logger log = new Logger(VectorExprSanityTest.class); + private static final int NUM_ITERATIONS = 10; private static final int VECTOR_SIZE = 512; final Map types = ImmutableMap.builder() @@ -57,81 +60,6 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest .put("s2", ExprType.STRING) .build(); - @Test - public void testUnaryOperators() - { - final String[] functions = new String[]{"-"}; - final String[] templates = new String[]{"%sd1", "%sl1"}; - - testFunctions(types, templates, functions); - } - - @Test - public void testBinaryOperators() - { - final String[] functions = new String[]{"+", "-", "*", "/", "^", "%", ">", ">=", "<", "<=", "==", "!="}; - final String[] templates = new String[]{ - "d1 %s d2", - "d1 %s l1", - "l1 %s d1", - "l1 %s l2", - "1 %s l1", - "1.1 %s d1", - "l1 %s 1", - "d1 %s 1.1" - }; - - testFunctions(types, templates, functions); - } - - @Test - public void testBinaryOperatorTrees() - { - final String[][] argsArrays = new String[][]{new String[]{"+", "-"}, new String[]{"*", "/"}}; - final String[] templates = new String[]{ - "(d1 %s d2) %s d2", - "(d1 %s d2) %s l2", - "(d1 %s l1) %s d2", - "(d1 %s l1) %s l2", - "(l1 %s d1) %s d2", - "(l1 %s d1) %s l2", - "(l1 %s l2) %s l2", - "(1 %s l1) %s d2", - "(1.1 %s d1) %s d2", - "(1.1 %s d1) %s l2", - "(l1 %s 1) %s l2", - "(l1 %s 1) %s d2", - "(d1 %s 1.1) %s d2", - "(d1 %s 1.1) %s l1" - }; - - testFunctions(types, templates, argsArrays); - } - - @Test - public void testUnivariateFunctions() - { - final String[] functions = new String[]{"parse_long"}; - final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"}; - testFunctions(types, templates, functions); - } - - @Test - public void testUnivariateMathFunctions() - { - final String[] functions = new String[]{"atan", "cos", "cosh", "cot", "sin", "sinh", "tan", "tanh"}; - final String[] templates = new String[]{"%s(l1)", "%s(d1)"}; - testFunctions(types, templates, functions); - } - - @Test - public void testBivariateMathFunctions() - { - final String[] functions = new String[]{"max", "min"}; - final String[] templates = new String[]{"%s(d1, d2)", "%s(d1, l1)", "%s(l1, d1)", "%s(l1, l2)"}; - testFunctions(types, templates, functions); - } - static void testFunctions(Map types, String[] templates, String[] args) { for (String template : templates) { @@ -154,6 +82,7 @@ static void testFunctions(Map types, String[] templates, Strin static void testExpression(String expr, Map types) { + log.debug("[%s]", expr); Expr parsed = Parser.parse(expr, ExprMacroTable.nil()); NonnullPair bindings; @@ -171,7 +100,7 @@ private static void testExpressionWithBindings( NonnullPair bindings ) { - Assert.assertTrue(parsed.canVectorize(bindings.rhs)); + Assert.assertTrue(StringUtils.format("Cannot vectorize %s", expr), parsed.canVectorize(bindings.rhs)); ExprType outputType = parsed.getOutputType(bindings.rhs); VectorExprEval vectorEval = parsed.buildVectorized(bindings.rhs).evalVector(bindings.rhs); Assert.assertEquals(outputType, vectorEval.getType()); @@ -180,11 +109,18 @@ private static void testExpressionWithBindings( if (!eval.isNumericNull()) { Assert.assertEquals(outputType, eval.type()); } - Assert.assertEquals(StringUtils.format("Values do not match for row %s for expression %s", i, expr), eval.value(), vectorEval.get(i)); + Assert.assertEquals( + StringUtils.format("Values do not match for row %s for expression %s", i, expr), + eval.value(), + vectorEval.get(i) + ); } } - static NonnullPair makeRandomizedBindings(int vectorSize, Map types) + static NonnullPair makeRandomizedBindings( + int vectorSize, + Map types + ) { final ThreadLocalRandom r = ThreadLocalRandom.current(); @@ -198,7 +134,10 @@ static NonnullPair makeRandomized ); } - static NonnullPair makeSequentialBinding(int vectorSize, Map types) + static NonnullPair makeSequentialBinding( + int vectorSize, + Map types + ) { return makeBindings( @@ -207,6 +146,7 @@ static NonnullPair makeSequential new LongSupplier() { int counter = 1; + @Override public long getAsLong() { @@ -216,6 +156,7 @@ public long getAsLong() new DoubleSupplier() { int counter = 1; + @Override public double getAsDouble() { @@ -226,6 +167,7 @@ public double getAsDouble() new Supplier() { int counter = 1; + @Override public String get() { @@ -278,7 +220,11 @@ static NonnullPair makeBindings( } objectBindings[i].withBinding(entry.getKey(), nulls[i] ? null : doubles[i]); } - vectorBinding.addDouble(entry.getKey(), doubles); + if (hasNulls) { + vectorBinding.addDouble(entry.getKey(), doubles, nulls); + } else { + vectorBinding.addDouble(entry.getKey(), doubles); + } break; case STRING: String[] strings = new String[vectorSize]; @@ -298,6 +244,94 @@ static NonnullPair makeBindings( return new NonnullPair<>(objectBindings, vectorBinding); } + static String[][] makeTemplateArgs(String[] arg1, String[] arg2) + { + return Arrays.stream(arg1) + .flatMap(a1 -> Arrays.stream(arg2).map(a2 -> new String[]{a1, a2})) + .toArray(String[][]::new); + } + + static String[][] makeTemplateArgs(String[] arg1, String[] arg2, String[] arg3) + { + return Arrays.stream(arg1) + .flatMap(a1 -> + Arrays.stream(arg2).flatMap(a2 -> Arrays.stream(arg3).map(a3 -> new String[]{a1, a2, a3})) + ) + .toArray(String[][]::new); + } + + @Test + public void testUnaryOperators() + { + final String[] functions = new String[]{"-"}; + final String[] templates = new String[]{"%sd1", "%sl1"}; + + testFunctions(types, templates, functions); + } + + @Test + public void testBinaryOperators() + { + final String[] columns = new String[]{"d1", "d2", "l1", "l2", "1", "1.0"}; + final String[][] templateInputs = makeTemplateArgs(columns, columns); + final String[] templates = + Arrays.stream(templateInputs) + .map(i -> StringUtils.format("%s %s %s", i[0], "%s", i[1])) + .toArray(String[]::new); + final String[] args = new String[]{"+", "-", "*", "/", "^", "%", ">", ">=", "<", "<=", "==", "!="}; + + testFunctions(types, templates, args); + } + + @Test + public void testBinaryOperatorTrees() + { + final String[] columns = new String[]{"d1", "l1", "1", "1.0"}; + final String[] columns2 = new String[]{"d2", "l2", "2", "2.0"}; + final String[][] templateInputs = makeTemplateArgs(columns, columns2, columns); + final String[] templates = + Arrays.stream(templateInputs) + .map(i -> StringUtils.format("(%s %s %s) %s %s", i[0], "%s", i[1], "%s", i[2])) + .toArray(String[]::new); + final String[] ops = new String[]{"+", "-", "*", "/"}; + final String[][] args = makeTemplateArgs(ops, ops); + testFunctions(types, templates, args); + } + + @Test + public void testUnivariateFunctions() + { + final String[] functions = new String[]{"parse_long"}; + final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testUnivariateMathFunctions() + { + final String[] functions = new String[]{"atan", "cos", "cosh", "cot", "sin", "sinh", "tan", "tanh"}; + final String[] templates = new String[]{"%s(l1)", "%s(d1)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testBivariateMathFunctions() + { + final String[] functions = new String[]{"max", "min"}; + final String[] templates = new String[]{"%s(d1, d2)", "%s(d1, l1)", "%s(l1, d1)", "%s(l1, l2)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testCast() + { + final String[] columns = new String[]{"d1", "l1", "s1"}; + final String[] castTo = new String[]{"'STRING'", "'LONG'", "'DOUBLE'"}; + final String[][] args = makeTemplateArgs(columns, castTo); + final String[] templates = new String[]{"cast(%s, %s)"}; + testFunctions(types, templates, args); + } + static class SettableObjectBinding implements Expr.ObjectBinding { private final Map bindings; @@ -347,6 +381,7 @@ public SettableVectorInputBinding addBinding(String name, ExprType type, boolean this.types.put(name, type); return this; } + public SettableVectorInputBinding addLong(String name, long[] longs) { return addLong(name, longs, new boolean[longs.length]); From c56095c87e26ebdfc49bc2132d3908c9dcf349c2 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 17 Sep 2020 09:52:36 -0700 Subject: [PATCH 07/17] what is an expression planner? --- .../java/org/apache/druid/math/expr/Expr.java | 8 + .../BivariateFunctionVectorProcessor.java | 7 +- .../vector/CastToDoubleVectorProcessor.java | 2 +- .../vector/CastToLongVectorProcessor.java | 2 +- .../vector/CastToStringVectorProcessor.java | 2 +- ...ubleUnivariateFunctionVectorProcessor.java | 6 - .../expr/vector/DoubleVectorExprEval.java | 2 +- ...ubleUnivariateFunctionVectorProcessor.java | 6 - ...LongUnivariateFunctionVectorProcessor.java | 6 - .../math/expr/vector/LongVectorExprEval.java | 2 +- .../expr/vector/StringVectorExprEval.java | 2 +- .../UnivariateFunctionVectorProcessor.java | 9 +- .../math/expr/vector/VectorExprEval.java | 6 + .../math/expr/vector/VectorExprProcessor.java | 4 + .../query/aggregation/AggregatorUtil.java | 3 +- .../segment/filter/ExpressionFilter.java | 1 + ...xpressionMultiValueDimensionSelector.java} | 64 +++- .../druid/segment/virtual/ExpressionPlan.java | 177 +++++++++ .../segment/virtual/ExpressionPlanner.java | 180 +++++++++ .../segment/virtual/ExpressionSelectors.java | 357 ++---------------- ...xpressionSingleValueDimensionSelector.java | 90 +++++ .../virtual/ExpressionVectorInputBinding.java | 107 ++++++ .../virtual/ExpressionVectorSelectors.java | 97 +++++ .../virtual/ExpressionVirtualColumn.java | 60 +-- ...RowBasedExpressionColumnValueSelector.java | 17 +- .../virtual/ExpressionVirtualColumnTest.java | 2 +- 26 files changed, 807 insertions(+), 412 deletions(-) rename processing/src/main/java/org/apache/druid/segment/virtual/{MultiValueExpressionDimensionSelector.java => ExpressionMultiValueDimensionSelector.java} (72%) create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSingleValueDimensionSelector.java create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java create mode 100644 processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index 06c6f047eaef..f5cd8e4d43b5 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -209,6 +209,9 @@ default boolean canVectorize(Expr... args) } } + /** + * {@link InputBindingTypes} + vectorizations stuff for {@link #buildVectorized} + */ interface VectorInputBindingTypes extends InputBindingTypes { int getMaxVectorSize(); @@ -226,6 +229,11 @@ interface ObjectBinding Object get(String name); } + /** + * Mechanism to supply batches of input values to a {@link VectorExprProcessor} for optimized processing. Mirrors + * the vectorized column selector interfaces, and includes {@link ExprType} information about all input bindings + * which exist + */ interface VectorInputBinding extends VectorInputBindingTypes { T[] getObjectVector(String name); diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java index db577be83104..52bb59b020fb 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java @@ -21,6 +21,11 @@ import org.apache.druid.math.expr.Expr; +/** + * common machinery for processing two input operators and functions, which should always treat null inputs as null + * output, and are backed by a primitive values instead of an object values (and need to use the null vectors instead of + * checking the vector themselves for nulls) + */ public abstract class BivariateFunctionVectorProcessor implements VectorExprProcessor { final VectorExprProcessor left; @@ -64,8 +69,6 @@ public final VectorExprEval evalVector(Expr.VectorInputBinding bindings outNulls[i] = (hasLeftNulls && leftNulls[i]) || (hasRightNulls && rightNulls[i]); if (!outNulls[i]) { processIndex(leftInput, rightInput, i); - } else { - processNull(i); } } } else { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java index a790101286cf..2afc6bdfe611 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java @@ -22,7 +22,7 @@ import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; -public class CastToDoubleVectorProcessor extends CastToTypeVectorProcessor +public final class CastToDoubleVectorProcessor extends CastToTypeVectorProcessor { public CastToDoubleVectorProcessor(VectorExprProcessor delegate) { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java index 8a725b377126..614e14a9210a 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java @@ -22,7 +22,7 @@ import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; -public class CastToLongVectorProcessor extends CastToTypeVectorProcessor +public final class CastToLongVectorProcessor extends CastToTypeVectorProcessor { public CastToLongVectorProcessor(VectorExprProcessor delegate) { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java index abf570def8d8..e5e09098810d 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java @@ -22,7 +22,7 @@ import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; -public class CastToStringVectorProcessor extends CastToTypeVectorProcessor +public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor { public CastToStringVectorProcessor(VectorExprProcessor delegate) { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java index 47c66d3577fa..415462017367 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java @@ -42,12 +42,6 @@ final void processIndex(double[] input, int i) outValues[i] = apply(input[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java index bc15a477ae7d..dd2ef0a39da2 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java @@ -24,7 +24,7 @@ import java.util.Arrays; -public class DoubleVectorExprEval extends VectorExprEval +public final class DoubleVectorExprEval extends VectorExprEval { public DoubleVectorExprEval(double[] values, boolean[] nulls) { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java index 15eb77d95520..292444313b89 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java @@ -42,12 +42,6 @@ final void processIndex(long[] input, int i) outValues[i] = apply(input[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java index 16897a7e070c..15ea8f0abee4 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java @@ -41,12 +41,6 @@ final void processIndex(long[] input, int i) { outValues[i] = apply(input[i]); } - - @Override - void processNull(int i) - { - outValues[i] = 0; - } @Override final VectorExprEval asEval() diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java index 47ab7509af0d..ec0648f3cb6c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java @@ -25,7 +25,7 @@ import javax.annotation.Nullable; import java.util.Arrays; -public class LongVectorExprEval extends VectorExprEval +public final class LongVectorExprEval extends VectorExprEval { public LongVectorExprEval(long[] values, @Nullable boolean[] nulls) { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java index 94f5ee0d1bc9..c8536571c5e4 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java @@ -26,7 +26,7 @@ import javax.annotation.Nullable; -public class StringVectorExprEval extends VectorExprEval +public final class StringVectorExprEval extends VectorExprEval { @Nullable private long[] longs; diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java index 1b5aca533dfe..10cde10b4c17 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java @@ -21,6 +21,11 @@ import org.apache.druid.math.expr.Expr; +/** + * common machinery for processing single input operators and functions, which should always treat null input as null + * output, and are backed by a primitive value instead of an object value (and need to use the null vector instead of + * checking the vector itself for nulls) + */ public abstract class UnivariateFunctionVectorProcessor implements VectorExprProcessor { final VectorExprProcessor processor; @@ -56,8 +61,6 @@ public final VectorExprEval evalVector(Expr.VectorInputBinding bindings outNulls[i] = inputNulls[i]; if (!outNulls[i]) { processIndex(input, i); - } else { - processNull(i); } } } else { @@ -71,7 +74,5 @@ public final VectorExprEval evalVector(Expr.VectorInputBinding bindings abstract void processIndex(TInput input, int i); - abstract void processNull(int i); - abstract VectorExprEval asEval(); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java index 17b190410ce3..63bf50cfa987 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java @@ -25,6 +25,12 @@ import javax.annotation.Nullable; import java.lang.reflect.Array; +/** + * Result of {@link VectorExprProcessor#evalVector} which wraps the actual evaluated results of the operation over the + * input vector(s). Methods to get actual results mirror vectorized value and object selectors. + * + * The generic parameter T should be the native java array type of the vector result (long[], String[], etc.) + */ public abstract class VectorExprEval { final T values; diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java index b17933b6a04c..88c2d672f2f3 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java @@ -22,6 +22,10 @@ import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; +/** + * Interface describing vectorized expression processors, which can be specialized using input type information to + * produce optimized expression evaluators, which can operate on batches of primitive data with minimal object overhead + */ public interface VectorExprProcessor { VectorExprEval evalVector(Expr.VectorInputBinding bindings); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index 8e191ea03051..dfa6ad051b30 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -34,6 +34,7 @@ import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; import org.apache.druid.segment.virtual.ExpressionSelectors; +import org.apache.druid.segment.virtual.ExpressionVectorSelectors; import javax.annotation.Nullable; import java.util.ArrayList; @@ -239,7 +240,7 @@ public static VectorValueSelector makeVectorValueSelector( throw new IllegalArgumentException("Only one of fieldName or expression should be non-null"); } if (expression != null) { - return ExpressionSelectors.makeVectorValueSelector(columnSelectorFactory, fieldExpression.get()); + return ExpressionVectorSelectors.makeVectorValueSelector(columnSelectorFactory, fieldExpression.get()); } return columnSelectorFactory.makeValueSelector(fieldName); } diff --git a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java index acf0dbeaf04e..51cfbf56c312 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/ExpressionFilter.java @@ -109,6 +109,7 @@ public boolean supportsBitmapIndex(final BitmapIndexSelector selector) { final Expr.BindingAnalysis details = this.bindingDetails.get(); + if (details.getRequiredBindings().isEmpty()) { // Constant expression. return true; diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/MultiValueExpressionDimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionMultiValueDimensionSelector.java similarity index 72% rename from processing/src/main/java/org/apache/druid/segment/virtual/MultiValueExpressionDimensionSelector.java rename to processing/src/main/java/org/apache/druid/segment/virtual/ExpressionMultiValueDimensionSelector.java index 513631e5e791..7f9f8c958af6 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/MultiValueExpressionDimensionSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionMultiValueDimensionSelector.java @@ -22,6 +22,7 @@ import com.google.common.base.Predicate; import org.apache.druid.common.config.NullHandling; import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.filter.ValueMatcher; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.ColumnValueSelector; @@ -41,11 +42,22 @@ * Basic multi-value dimension selector for an {@link org.apache.druid.math.expr.Expr} evaluating * {@link ColumnValueSelector}. */ -public class MultiValueExpressionDimensionSelector implements DimensionSelector +public class ExpressionMultiValueDimensionSelector implements DimensionSelector { - private final ColumnValueSelector baseSelector; + public static ExpressionMultiValueDimensionSelector fromValueSelector( + ColumnValueSelector baseSelector, + @Nullable ExtractionFn extractionFn + ) + { + if (extractionFn != null) { + return new ExtractionMultiValueDimensionSelector(baseSelector, extractionFn); + } + return new ExpressionMultiValueDimensionSelector(baseSelector); + } - public MultiValueExpressionDimensionSelector(ColumnValueSelector baseSelector) + protected final ColumnValueSelector baseSelector; + + public ExpressionMultiValueDimensionSelector(ColumnValueSelector baseSelector) { this.baseSelector = baseSelector; } @@ -196,4 +208,50 @@ public Class classOfObject() { return Object.class; } + + /** + * expressions + extractions + */ + static class ExtractionMultiValueDimensionSelector extends ExpressionMultiValueDimensionSelector + { + private final ExtractionFn extractionFn; + + private ExtractionMultiValueDimensionSelector(ColumnValueSelector baseSelector, ExtractionFn extractionFn) + { + super(baseSelector); + this.extractionFn = extractionFn; + } + + @Override + String getValue(ExprEval evaluated) + { + assert !evaluated.isArray(); + return extractionFn.apply(NullHandling.emptyToNullIfNeeded(evaluated.asString())); + } + + @Override + List getArray(ExprEval evaluated) + { + assert evaluated.isArray(); + return Arrays.stream(evaluated.asStringArray()) + .map(x -> extractionFn.apply(NullHandling.emptyToNullIfNeeded(x))) + .collect(Collectors.toList()); + } + + @Override + String getArrayValue(ExprEval evaluated, int i) + { + assert evaluated.isArray(); + String[] stringArray = evaluated.asStringArray(); + assert i < stringArray.length; + return extractionFn.apply(NullHandling.emptyToNullIfNeeded(stringArray[i])); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseSelector", baseSelector); + inspector.visit("extractionFn", extractionFn); + } + } } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java new file mode 100644 index 000000000000..04481b7e5c77 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import com.google.common.collect.Iterables; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Parser; +import org.apache.druid.segment.column.ValueType; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +public class ExpressionPlan +{ + public enum Trait + { + /** + * expression has no inputs and can be optimized into a constant selector + */ + CONSTANT, + /** + * expression has a single, single valued input, and is dictionary encoded if the value is a string + */ + SINGLE_INPUT_SCALAR, + /** + * expression has a single input, which may produce single or multi-valued output, but if so, it must be implicitly + * mappable (i.e. the expression is not treating its input as an array and not wanting to output an array) + */ + SINGLE_INPUT_MAPPABLE, + /** + * expression must be implicitly mapped across the multiple values per row of known multi-value inputs + */ + NEEDS_APPLIED, + /** + * expression has inputs whose type was unresolveable, or was incomplete, such as unknown multi-valuedness + */ + UNKNOWN_INPUTS, + /** + * expression explicitly using multi-valued inputs as array inputs + */ + NON_SCALAR_INPUTS, + /** + * expression produces explict multi-valued output, or implicit multi-valued output via mapping + */ + NON_SCALAR_OUTPUT, + /** + * expression is vectorizable + */ + VECTORIZABLE + } + + private final Expr expression; + private final Expr.BindingAnalysis analysis; + private final EnumSet traits; + + @Nullable + private final ExprType outputType; + @Nullable + private final ValueType singleInputType; + private final Set unknownInputs; + private final List unappliedInputs; + + ExpressionPlan( + Expr expression, + Expr.BindingAnalysis analysis, + EnumSet traits, + @Nullable ExprType outputType, + @Nullable ValueType singleInputType, + Set unknownInputs, + List unappliedInputs + ) + { + this.expression = expression; + this.analysis = analysis; + this.traits = traits; + this.outputType = outputType; + this.singleInputType = singleInputType; + this.unknownInputs = unknownInputs; + this.unappliedInputs = unappliedInputs; + } + + public Expr getExpression() + { + return expression; + } + + public Expr getAppliedExpression() + { + if (is(Trait.NEEDS_APPLIED)) { + return Parser.applyUnappliedBindings(expression, analysis, unappliedInputs); + } + return expression; + } + + public Expr.BindingAnalysis getAnalysis() + { + return analysis; + } + + public boolean is(Trait... flags) + { + return is(traits, flags); + } + + public boolean any(Trait... flags) + { + return any(traits, flags); + } + + public boolean none(Trait... flags) + { + return none(traits, flags); + } + + @Nullable + public ExprType getOutputType() + { + return outputType; + } + + @Nullable + public ValueType getSingleInputType() + { + return singleInputType; + } + + public String getSingleInputName() + { + return Iterables.getOnlyElement(analysis.getRequiredBindings()); + } + + public Set getUnknownInputs() + { + return unknownInputs; + } + + public List getUnappliedInputs() + { + return unappliedInputs; + } + + static boolean is(EnumSet traits, Trait... args) + { + return Arrays.stream(args).allMatch(traits::contains); + } + + static boolean any(EnumSet traits, Trait... args) + { + return Arrays.stream(args).anyMatch(traits::contains); + } + + static boolean none(EnumSet traits, Trait... args) + { + return Arrays.stream(args).noneMatch(traits::contains); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java new file mode 100644 index 000000000000..dc6361e746b4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlanner.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Parser; +import org.apache.druid.segment.ColumnInspector; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ValueType; + +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class ExpressionPlanner +{ + private ExpressionPlanner() + { + // No instantiation. + } + + /** + * Druid tries to be chill to expressions to make up for not having a well defined table schema across segments. This + * method performs some analysis to determine what sort of selectors can be constructed on top of an expression, + * whether or not the expression will need implicitly mapped across multi-valued inputs, if the expression produces + * multi-valued outputs, is vectorizable, and everything else interesting when making a selector. + * + * Results are stored in a {@link ExpressionPlan}, which can be examined to do whatever is necessary to make things + * function properly. + */ + public static ExpressionPlan plan(ColumnInspector inspector, Expr expression) + { + final Expr.BindingAnalysis analysis = expression.analyzeInputs(); + Parser.validateExpr(expression, analysis); + + EnumSet traits = EnumSet.noneOf(ExpressionPlan.Trait.class); + Set maybeMultiValued = new HashSet<>(); + List needsApplied = ImmutableList.of(); + ValueType singleInputType = null; + ExprType outputType = null; + + final Set columns = analysis.getRequiredBindings(); + + // check and set traits which allow optimized selectors to be created + if (columns.isEmpty()) { + traits.add(ExpressionPlan.Trait.CONSTANT); + } else if (columns.size() == 1) { + final String column = Iterables.getOnlyElement(columns); + final ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); + + // These flags allow for selectors that wrap a single underlying column to be optimized, through caching results + // and via allowing deferred execution in the case of building dimension selectors. + // SINGLE_INPUT_SCALAR + // is set if an input is single valued, and the output is definitely single valued, with an additional requirement + // for strings that the column is dictionary encoded. + // SINGLE_INPUT_MAPPABLE + // is set when a single input string column, which can be multi-valued, but if so, it must be implicitly mappable + // (i.e. the expression is not treating its input as an array and not wanting to output an array) + if (capabilities != null) { + boolean isSingleInputMappable = false; + boolean isSingleInputScalar = capabilities.hasMultipleValues().isFalse() && + !analysis.hasInputArrays() && + !analysis.isOutputArray(); + if (capabilities.getType() == ValueType.STRING) { + isSingleInputScalar &= capabilities.isDictionaryEncoded().isTrue(); + isSingleInputMappable = capabilities.isDictionaryEncoded().isTrue() && + !capabilities.hasMultipleValues().isUnknown() && + !analysis.hasInputArrays() && + !analysis.isOutputArray(); + } + + // if satisfied, set single input output type and flags + if (isSingleInputScalar || isSingleInputMappable) { + singleInputType = capabilities.getType(); + if (isSingleInputScalar) { + traits.add(ExpressionPlan.Trait.SINGLE_INPUT_SCALAR); + } + if (isSingleInputMappable) { + traits.add(ExpressionPlan.Trait.SINGLE_INPUT_MAPPABLE); + } + } + } + } + + // if we didn't eliminate this expression as a single input scalar or mappable expression, it might need + // automatic transformation to map across multi-valued inputs (or row by row detection in the worst case) + if (ExpressionPlan.none(traits, ExpressionPlan.Trait.SINGLE_INPUT_SCALAR)) { + final Set definitelyMultiValued = new HashSet<>(); + for (String column : analysis.getRequiredBindings()) { + final ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); + if (capabilities != null) { + if (capabilities.hasMultipleValues().isTrue()) { + definitelyMultiValued.add(column); + } else if (capabilities.getType().equals(ValueType.STRING) && + capabilities.hasMultipleValues().isMaybeTrue() && + !analysis.getArrayBindings().contains(column) + ) { + maybeMultiValued.add(column); + } + } else { + maybeMultiValued.add(column); + } + } + + // find any inputs which will need implicitly mapped across multi-valued rows + needsApplied = + columns.stream() + .filter(c -> definitelyMultiValued.contains(c) && !analysis.getArrayBindings().contains(c)) + .collect(Collectors.toList()); + + // if any multi-value inputs, set flag for non-scalar inputs + if (analysis.hasInputArrays()) { + traits.add(ExpressionPlan.Trait.NON_SCALAR_INPUTS); + } + + if (!maybeMultiValued.isEmpty()) { + traits.add(ExpressionPlan.Trait.UNKNOWN_INPUTS); + } + + // if expression needs transformed, lets do it + if (!needsApplied.isEmpty()) { + traits.add(ExpressionPlan.Trait.NEEDS_APPLIED); + } + } + + // only set output type + if (ExpressionPlan.none(traits, ExpressionPlan.Trait.UNKNOWN_INPUTS, ExpressionPlan.Trait.NEEDS_APPLIED)) { + outputType = expression.getOutputType(inspector); + } + + // if analysis, inferred output type, or implicit mapping is in play, output will be multi-valued + if (analysis.isOutputArray() || ExprType.isArray(outputType) || ExpressionPlan.is(traits, ExpressionPlan.Trait.NEEDS_APPLIED)) { + traits.add(ExpressionPlan.Trait.NON_SCALAR_OUTPUT); + } + + // vectorized expressions do not currently support unknown inputs, multi-valued inputs or outputs, implicit mapping + boolean supportsVector = ExpressionPlan.none( + traits, + ExpressionPlan.Trait.UNKNOWN_INPUTS, + ExpressionPlan.Trait.NEEDS_APPLIED, + ExpressionPlan.Trait.NON_SCALAR_INPUTS, + ExpressionPlan.Trait.NON_SCALAR_OUTPUT + ); + + if (supportsVector && expression.canVectorize(inspector)) { + traits.add(ExpressionPlan.Trait.VECTORIZABLE); + } + return new ExpressionPlan( + expression, + analysis, + traits, + outputType, + singleInputType, + maybeMultiValued, + needsApplied + ); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java index 03c763186c86..741fc0684a86 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSelectors.java @@ -24,20 +24,15 @@ import com.google.common.base.Supplier; import com.google.common.collect.Iterables; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.UOE; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; -import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.Parser; -import org.apache.druid.math.expr.vector.VectorExprProcessor; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.expression.ExprUtils; import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.BaseObjectColumnValueSelector; -import org.apache.druid.segment.BaseSingleValueDimensionSelector; -import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.ConstantExprEvalSelector; @@ -47,19 +42,12 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.data.IndexedInts; -import org.apache.druid.segment.vector.NilVectorSelector; -import org.apache.druid.segment.vector.VectorColumnSelectorFactory; -import org.apache.druid.segment.vector.VectorObjectSelector; -import org.apache.druid.segment.vector.VectorSizeInspector; -import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; public class ExpressionSelectors @@ -134,32 +122,6 @@ public void inspectRuntimeShape(RuntimeShapeInspector inspector) }; } - - - public static VectorValueSelector makeVectorValueSelector( - VectorColumnSelectorFactory factory, - Expr expression - ) - { - final Expr.BindingAnalysis exprAnalysis = expression.analyzeInputs(); - Parser.validateExpr(expression, exprAnalysis); - final Expr.VectorInputBinding bindings = createVectorBindings(exprAnalysis, factory); - final VectorExprProcessor processor = expression.buildVectorized(bindings); - return new ExpressionVectorValueSelector(processor, bindings); - } - - public static VectorObjectSelector makeVectorObjectSelector( - VectorColumnSelectorFactory factory, - Expr expression - ) - { - final Expr.BindingAnalysis exprAnalysis = expression.analyzeInputs(); - Parser.validateExpr(expression, exprAnalysis); - final Expr.VectorInputBinding bindings = createVectorBindings(exprAnalysis, factory); - final VectorExprProcessor processor = expression.buildVectorized(bindings); - return new ExpressionVectorObjectSelector(processor, bindings); - } - /** * Makes a ColumnValueSelector whose getObject method returns an {@link ExprEval}. * @@ -170,69 +132,45 @@ public static ColumnValueSelector makeExprEvalSelector( Expr expression ) { - final Expr.BindingAnalysis bindingAnalysis = expression.analyzeInputs(); - Parser.validateExpr(expression, bindingAnalysis); - final List columns = bindingAnalysis.getRequiredBindingsList(); - - if (columns.size() == 1) { - final String column = Iterables.getOnlyElement(columns); - final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(column); + return makeExprEvalSelector(columnSelectorFactory, ExpressionPlanner.plan(columnSelectorFactory, expression)); + } - if (capabilities != null && capabilities.getType() == ValueType.LONG) { - // Optimization for expressions that hit one long column and nothing else. + public static ColumnValueSelector makeExprEvalSelector( + ColumnSelectorFactory columnSelectorFactory, + ExpressionPlan plan + ) + { + if (plan.is(ExpressionPlan.Trait.SINGLE_INPUT_SCALAR)) { + final String column = plan.getSingleInputName(); + final ValueType inputType = plan.getSingleInputType(); + if (inputType == ValueType.LONG) { return new SingleLongInputCachingExpressionColumnValueSelector( columnSelectorFactory.makeColumnValueSelector(column), - expression, + plan.getExpression(), !ColumnHolder.TIME_COLUMN_NAME.equals(column) // __time doesn't need an LRU cache since it is sorted. ); - } else if (capabilities != null - && capabilities.getType() == ValueType.STRING - && capabilities.isDictionaryEncoded().isTrue() - && capabilities.hasMultipleValues().isFalse() - && bindingAnalysis.getArrayBindings().isEmpty()) { - // Optimization for expressions that hit one scalar string column and nothing else. + } else if (inputType == ValueType.STRING) { return new SingleStringInputCachingExpressionColumnValueSelector( columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(column, column, ValueType.STRING)), - expression + plan.getExpression() ); } } + final Expr.ObjectBinding bindings = createBindings(plan.getAnalysis(), columnSelectorFactory); - final Pair, Set> arrayUsage = findArrayInputBindings(columnSelectorFactory, bindingAnalysis); - final Set actualArrays = arrayUsage.lhs; - final Set unknownIfArrays = arrayUsage.rhs; - - final List needsApplied = - columns.stream() - .filter(c -> actualArrays.contains(c) && !bindingAnalysis.getArrayBindings().contains(c)) - .collect(Collectors.toList()); - final Expr finalExpr; - if (needsApplied.size() > 0) { - finalExpr = Parser.applyUnappliedBindings(expression, bindingAnalysis, needsApplied); - } else { - finalExpr = expression; - } - - final Expr.ObjectBinding bindings = createBindings(bindingAnalysis, columnSelectorFactory); - + // Optimization for constant expressions if (bindings.equals(ExprUtils.nilBindings())) { - // Optimization for constant expressions. - return new ConstantExprEvalSelector(expression.eval(bindings)); + return new ConstantExprEvalSelector(plan.getExpression().eval(bindings)); } // if any unknown column input types, fall back to an expression selector that examines input bindings on a // per row basis - if (unknownIfArrays.size() > 0) { - return new RowBasedExpressionColumnValueSelector( - finalExpr, - bindingAnalysis, - bindings, - unknownIfArrays - ); + if (plan.is(ExpressionPlan.Trait.UNKNOWN_INPUTS)) { + return new RowBasedExpressionColumnValueSelector(plan, bindings); } // generic expression value selector for fully known input types - return new ExpressionColumnValueSelector(finalExpr, bindings); + return new ExpressionColumnValueSelector(plan.getAppliedExpression(), bindings); } /** @@ -245,38 +183,19 @@ public static DimensionSelector makeDimensionSelector( @Nullable final ExtractionFn extractionFn ) { - final Expr.BindingAnalysis bindingAnalysis = expression.analyzeInputs(); - Parser.validateExpr(expression, bindingAnalysis); - final List columns = bindingAnalysis.getRequiredBindingsList(); + final ExpressionPlan plan = ExpressionPlanner.plan(columnSelectorFactory, expression); - if (columns.size() == 1) { - final String column = Iterables.getOnlyElement(columns); - final ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(column); - - // Optimization for dimension selectors that wrap a single underlying string column. - // The string column can be multi-valued, but if so, it must be implicitly mappable (i.e. the expression is - // not treating it as an array and not wanting to output an array - if (capabilities != null - && capabilities.getType() == ValueType.STRING - && capabilities.isDictionaryEncoded().isTrue() - && canMapOverDictionary(bindingAnalysis, capabilities.hasMultipleValues()) - ) { + if (plan.is(ExpressionPlan.Trait.SINGLE_INPUT_MAPPABLE)) { + final String column = plan.getSingleInputName(); + if (plan.getSingleInputType() == ValueType.STRING) { return new SingleStringInputDimensionSelector( - columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(column, column, ValueType.STRING)), + columnSelectorFactory.makeDimensionSelector(DefaultDimensionSpec.of(column)), expression ); } } - final Pair, Set> arrayUsage = findArrayInputBindings(columnSelectorFactory, bindingAnalysis); - final Set actualArrays = arrayUsage.lhs; - final Set unknownIfArrays = arrayUsage.rhs; - - final ColumnValueSelector baseSelector = makeExprEvalSelector(columnSelectorFactory, expression); - final boolean multiVal = actualArrays.size() > 0 || - bindingAnalysis.getArrayBindings().size() > 0 || - unknownIfArrays.size() > 0; if (baseSelector instanceof ConstantExprEvalSelector) { // Optimization for dimension selectors on constants. @@ -284,87 +203,15 @@ && canMapOverDictionary(bindingAnalysis, capabilities.hasMultipleValues()) } else if (baseSelector instanceof NilColumnValueSelector) { // Optimization for null dimension selector. return DimensionSelector.constant(null); - } else if (extractionFn == null) { - - if (multiVal) { - return new MultiValueExpressionDimensionSelector(baseSelector); - } else { - class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelector - { - @Override - protected String getValue() - { - return NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString()); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("baseSelector", baseSelector); - } - } - return new DefaultExpressionDimensionSelector(); - } } else { - if (multiVal) { - class ExtractionMultiValueDimensionSelector extends MultiValueExpressionDimensionSelector - { - private ExtractionMultiValueDimensionSelector() - { - super(baseSelector); - } - - @Override - String getValue(ExprEval evaluated) - { - assert !evaluated.isArray(); - return extractionFn.apply(NullHandling.emptyToNullIfNeeded(evaluated.asString())); - } - - @Override - List getArray(ExprEval evaluated) - { - assert evaluated.isArray(); - return Arrays.stream(evaluated.asStringArray()) - .map(x -> extractionFn.apply(NullHandling.emptyToNullIfNeeded(x))) - .collect(Collectors.toList()); - } - - @Override - String getArrayValue(ExprEval evaluated, int i) - { - assert evaluated.isArray(); - String[] stringArray = evaluated.asStringArray(); - assert i < stringArray.length; - return extractionFn.apply(NullHandling.emptyToNullIfNeeded(stringArray[i])); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("baseSelector", baseSelector); - inspector.visit("extractionFn", extractionFn); - } - } - return new ExtractionMultiValueDimensionSelector(); - + if (plan.any( + ExpressionPlan.Trait.NON_SCALAR_OUTPUT, + ExpressionPlan.Trait.NEEDS_APPLIED, + ExpressionPlan.Trait.UNKNOWN_INPUTS + )) { + return ExpressionMultiValueDimensionSelector.fromValueSelector(baseSelector, extractionFn); } else { - class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSelector - { - @Override - protected String getValue() - { - return extractionFn.apply(NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString())); - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("baseSelector", baseSelector); - inspector.visit("extractionFn", extractionFn); - } - } - return new ExtractionExpressionDimensionSelector(); + return ExpressionSingleValueDimensionSelector.fromValueSelector(baseSelector, extractionFn); } } } @@ -452,39 +299,6 @@ private static Expr.ObjectBinding createBindings( } } - private static Expr.VectorInputBinding createVectorBindings( - Expr.BindingAnalysis bindingAnalysis, - VectorColumnSelectorFactory vectorColumnSelectorFactory - ) - { - VectorSelectorsVectorInputBinding binding = new VectorSelectorsVectorInputBinding(vectorColumnSelectorFactory.getVectorSizeInspector()); - final List columns = bindingAnalysis.getRequiredBindingsList(); - for (String columnName : columns) { - final ColumnCapabilities columnCapabilities = vectorColumnSelectorFactory.getColumnCapabilities(columnName); - final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; - - // null capabilities should be backed by a nil vector selector since it means the column effectively doesnt exist - if (nativeType != null) { - switch (nativeType) { - case FLOAT: - case DOUBLE: - binding.addNumeric(columnName, ExprType.DOUBLE, vectorColumnSelectorFactory.makeValueSelector(columnName)); - break; - case LONG: - binding.addNumeric(columnName, ExprType.LONG, vectorColumnSelectorFactory.makeValueSelector(columnName)); - break; - default: - binding.addObjectSelector( - columnName, - ExprType.STRING, - vectorColumnSelectorFactory.makeObjectSelector(columnName) - ); - } - } - } - return binding; - } - /** * Wraps a {@link ColumnValueSelector} and uses it to supply numeric values in a null-aware way. * @@ -659,111 +473,4 @@ public static Object coerceEvalToSelectorObject(ExprEval eval) return eval.value(); } } - - /** - * Returns pair of columns which are definitely multi-valued, or 'actual' arrays, and those which we are unable to - * discern from the {@link ColumnInspector#getColumnCapabilities(String)}, or 'unknown' arrays. - */ - public static Pair, Set> findArrayInputBindings( - ColumnInspector inspector, - Expr.BindingAnalysis bindingAnalysis - ) - { - final Set actualArrays = new HashSet<>(); - final Set unknownIfArrays = new HashSet<>(); - for (String column : bindingAnalysis.getRequiredBindings()) { - final ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); - if (capabilities != null) { - if (capabilities.hasMultipleValues().isTrue()) { - actualArrays.add(column); - } else if ( - capabilities.getType().equals(ValueType.STRING) && - capabilities.hasMultipleValues().isMaybeTrue() && - !bindingAnalysis.getArrayBindings().contains(column) - ) { - unknownIfArrays.add(column); - } - } else { - unknownIfArrays.add(column); - } - } - - return new Pair<>(actualArrays, unknownIfArrays); - } - - static class VectorSelectorsVectorInputBinding implements Expr.VectorInputBinding - { - private final Map numeric; - private final Map objects; - private final Map types; - private final NilVectorSelector nilSelector; - - private final VectorSizeInspector sizeInspector; - - public VectorSelectorsVectorInputBinding(VectorSizeInspector sizeInspector) - { - this.numeric = new HashMap<>(); - this.objects = new HashMap<>(); - this.types = new HashMap<>(); - this.sizeInspector = sizeInspector; - this.nilSelector = NilVectorSelector.create(sizeInspector); - } - - public VectorSelectorsVectorInputBinding addNumeric(String name, ExprType type, VectorValueSelector selector) - { - numeric.put(name, selector); - types.put(name, type); - return this; - } - - public VectorSelectorsVectorInputBinding addObjectSelector(String name, ExprType type, VectorObjectSelector selector) - { - objects.put(name, selector); - types.put(name, type); - return this; - } - - @Override - public T[] getObjectVector(String name) - { - return (T[]) objects.getOrDefault(name, nilSelector).getObjectVector(); - } - - @Override - public ExprType getType(String name) - { - return types.get(name); - } - - @Override - public long[] getLongVector(String name) - { - return numeric.getOrDefault(name, nilSelector).getLongVector(); - } - - @Override - public double[] getDoubleVector(String name) - { - return numeric.getOrDefault(name, nilSelector).getDoubleVector(); - } - - @Nullable - @Override - public boolean[] getNullVector(String name) - { - return numeric.getOrDefault(name, nilSelector).getNullVector(); - } - - @Override - public int getMaxVectorSize() - { - return sizeInspector.getMaxVectorSize(); - } - - @Override - public int getCurrentVectorSize() - { - return sizeInspector.getCurrentVectorSize(); - } - } } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSingleValueDimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSingleValueDimensionSelector.java new file mode 100644 index 000000000000..0dd991d1d535 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionSingleValueDimensionSelector.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.query.extraction.ExtractionFn; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.BaseSingleValueDimensionSelector; +import org.apache.druid.segment.ColumnValueSelector; + +import javax.annotation.Nullable; + +class ExpressionSingleValueDimensionSelector extends BaseSingleValueDimensionSelector +{ + public static ExpressionSingleValueDimensionSelector fromValueSelector( + ColumnValueSelector baseSelector, + @Nullable ExtractionFn extractionFn + ) + { + if (extractionFn != null) { + return new ExtractionExpressionDimensionSelector(baseSelector, extractionFn); + } + return new ExpressionSingleValueDimensionSelector(baseSelector); + } + + protected final ColumnValueSelector baseSelector; + + ExpressionSingleValueDimensionSelector(ColumnValueSelector baseSelector) + { + this.baseSelector = baseSelector; + } + + @Override + protected String getValue() + { + return NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString()); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseSelector", baseSelector); + } + + + /** + * expressions + extractions + */ + static class ExtractionExpressionDimensionSelector extends ExpressionSingleValueDimensionSelector + { + private final ExtractionFn extractionFn; + + ExtractionExpressionDimensionSelector(ColumnValueSelector baseSelector, ExtractionFn extractionFn) + { + super(baseSelector); + this.extractionFn = extractionFn; + } + + @Override + protected String getValue() + { + return extractionFn.apply(NullHandling.emptyToNullIfNeeded(baseSelector.getObject().asString())); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("baseSelector", baseSelector); + inspector.visit("extractionFn", extractionFn); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java new file mode 100644 index 000000000000..d6b0846ed411 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorInputBinding.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.segment.vector.NilVectorSelector; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorSizeInspector; +import org.apache.druid.segment.vector.VectorValueSelector; + +import javax.annotation.Nullable; +import java.util.HashMap; +import java.util.Map; + +class ExpressionVectorInputBinding implements Expr.VectorInputBinding +{ + private final Map numeric; + private final Map objects; + private final Map types; + private final NilVectorSelector nilSelector; + + private final VectorSizeInspector sizeInspector; + + public ExpressionVectorInputBinding(VectorSizeInspector sizeInspector) + { + this.numeric = new HashMap<>(); + this.objects = new HashMap<>(); + this.types = new HashMap<>(); + this.sizeInspector = sizeInspector; + this.nilSelector = NilVectorSelector.create(sizeInspector); + } + + public ExpressionVectorInputBinding addNumeric(String name, ExprType type, VectorValueSelector selector) + { + numeric.put(name, selector); + types.put(name, type); + return this; + } + + public ExpressionVectorInputBinding addObjectSelector(String name, ExprType type, VectorObjectSelector selector) + { + objects.put(name, selector); + types.put(name, type); + return this; + } + + @Override + public T[] getObjectVector(String name) + { + return (T[]) objects.getOrDefault(name, nilSelector).getObjectVector(); + } + + @Override + public ExprType getType(String name) + { + return types.get(name); + } + + @Override + public long[] getLongVector(String name) + { + return numeric.getOrDefault(name, nilSelector).getLongVector(); + } + + @Override + public double[] getDoubleVector(String name) + { + return numeric.getOrDefault(name, nilSelector).getDoubleVector(); + } + + @Nullable + @Override + public boolean[] getNullVector(String name) + { + return numeric.getOrDefault(name, nilSelector).getNullVector(); + } + + @Override + public int getMaxVectorSize() + { + return sizeInspector.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return sizeInspector.getCurrentVectorSize(); + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java new file mode 100644 index 000000000000..bef524749165 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import com.google.common.base.Preconditions; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; + +import java.util.List; + +public class ExpressionVectorSelectors +{ + private ExpressionVectorSelectors() + { + // No instantiation. + } + + public static VectorValueSelector makeVectorValueSelector( + VectorColumnSelectorFactory factory, + Expr expression + ) + { + final ExpressionPlan plan = ExpressionPlanner.plan(factory, expression); + Preconditions.checkArgument(plan.is(ExpressionPlan.Trait.VECTORIZABLE)); + final Expr.VectorInputBinding bindings = createVectorBindings(plan.getAnalysis(), factory); + final VectorExprProcessor processor = plan.getExpression().buildVectorized(bindings); + return new ExpressionVectorValueSelector(processor, bindings); + } + + public static VectorObjectSelector makeVectorObjectSelector( + VectorColumnSelectorFactory factory, + Expr expression + ) + { + final ExpressionPlan plan = ExpressionPlanner.plan(factory, expression); + Preconditions.checkArgument(plan.is(ExpressionPlan.Trait.VECTORIZABLE)); + final Expr.VectorInputBinding bindings = createVectorBindings(plan.getAnalysis(), factory); + final VectorExprProcessor processor = plan.getExpression().buildVectorized(bindings); + return new ExpressionVectorObjectSelector(processor, bindings); + } + + private static Expr.VectorInputBinding createVectorBindings( + Expr.BindingAnalysis bindingAnalysis, + VectorColumnSelectorFactory vectorColumnSelectorFactory + ) + { + ExpressionVectorInputBinding binding = new ExpressionVectorInputBinding(vectorColumnSelectorFactory.getVectorSizeInspector()); + final List columns = bindingAnalysis.getRequiredBindingsList(); + for (String columnName : columns) { + final ColumnCapabilities columnCapabilities = vectorColumnSelectorFactory.getColumnCapabilities(columnName); + final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; + + // null capabilities should be backed by a nil vector selector since it means the column effectively doesnt exist + if (nativeType != null) { + switch (nativeType) { + case FLOAT: + case DOUBLE: + binding.addNumeric(columnName, ExprType.DOUBLE, vectorColumnSelectorFactory.makeValueSelector(columnName)); + break; + case LONG: + binding.addNumeric(columnName, ExprType.LONG, vectorColumnSelectorFactory.makeValueSelector(columnName)); + break; + default: + binding.addObjectSelector( + columnName, + ExprType.STRING, + vectorColumnSelectorFactory.makeObjectSelector(columnName) + ); + } + } + } + return binding; + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index 1e43f71e9bf7..8e637d9966be 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -27,7 +27,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprType; @@ -49,7 +48,6 @@ import javax.annotation.Nullable; import java.util.List; import java.util.Objects; -import java.util.Set; public class ExpressionVirtualColumn implements VirtualColumn { @@ -138,88 +136,64 @@ public ColumnValueSelector makeColumnValueSelector(String columnName, ColumnS return ExpressionSelectors.makeColumnValueSelector(factory, parsedExpression.get()); } - @Override public boolean canVectorize(ColumnInspector inspector) { - Expr expr = parsedExpression.get(); - Expr.BindingAnalysis analysis = expr.analyzeInputs(); - // we don't currently support multi-value inputs or outputs for vectorized expressions - return !analysis.hasInputArrays() && - !analysis.isOutputArray() && - analysis.getRequiredBindingsList().stream().noneMatch(column -> { - ColumnCapabilities capabilities = inspector.getColumnCapabilities(column); - return capabilities == null || capabilities.hasMultipleValues().isMaybeTrue(); - }) && - parsedExpression.get().canVectorize(inspector); + final ExpressionPlan plan = ExpressionPlanner.plan(inspector, parsedExpression.get()); + return plan.is(ExpressionPlan.Trait.VECTORIZABLE); } @Override public VectorValueSelector makeVectorValueSelector(String columnName, VectorColumnSelectorFactory factory) { - return ExpressionSelectors.makeVectorValueSelector(factory, parsedExpression.get()); + return ExpressionVectorSelectors.makeVectorValueSelector(factory, parsedExpression.get()); } @Override public VectorObjectSelector makeVectorObjectSelector(String columnName, VectorColumnSelectorFactory factory) { - return ExpressionSelectors.makeVectorObjectSelector(factory, parsedExpression.get()); + return ExpressionVectorSelectors.makeVectorObjectSelector(factory, parsedExpression.get()); } @Override public ColumnCapabilities capabilities(String columnName) { - // Note: Ideally we would fill out additional information instead of leaving capabilities as 'unknown', e.g. examine - // if the expression in question could potentially return multiple values and anything else. However, we don't - // currently have a good way of determining this, so fill this out more once we do + // If possible, this should only be used as a fallback method for when capabilities are truly 'unknown', because we + // are unable to compute the output type of the expression, either due to incomplete type information of the + // inputs or because of unimplemented methods on expression implementations themselves, or, because a + // ColumnInspector is not available return new ColumnCapabilitiesImpl().setType(outputType == null ? ValueType.FLOAT : outputType); } @Override public ColumnCapabilities capabilities(ColumnInspector inspector, String columnName) { - final ExprType inferredOutputType = parsedExpression.get().getOutputType(inspector); + final ExpressionPlan plan = ExpressionPlanner.plan(inspector, parsedExpression.get()); - if (inferredOutputType != null) { + if (plan.getOutputType() != null) { + final ExprType inferredOutputType = plan.getOutputType(); final ValueType valueType = ExprType.toValueType(inferredOutputType); if (valueType.isNumeric()) { // if float was explicitly specified preserve it, because it will currently never be the computed output type - if (ValueType.FLOAT.equals(outputType)) { + if (ValueType.FLOAT == outputType) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.FLOAT); } return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ExprType.toValueType(inferredOutputType)); } - if (valueType.isArray()) { - // always a multi-value string since wider engine does not yet support array types - return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(ValueType.STRING); - } - final Expr.BindingAnalysis analysis = parsedExpression.get().analyzeInputs(); - if (analysis.isOutputArray()) { + // we don't have to check for unknown input here because output type is unable to be inferred if we don't know + // the complete set of input types + if (plan.any(ExpressionPlan.Trait.NON_SCALAR_OUTPUT, ExpressionPlan.Trait.NEEDS_APPLIED)) { // always a multi-value string since wider engine does not yet support array types - return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(ValueType.STRING); + return new ColumnCapabilitiesImpl().setType(ValueType.STRING); } - // dupe of some logic in ExpressionSelectors to detect if any possiblility of this producing multi value outputs - final Pair, Set> arrayUsage = ExpressionSelectors.findArrayInputBindings(inspector, analysis); - final Set actualArrays = arrayUsage.lhs; - final Set unknownIfArrays = arrayUsage.rhs; - final long needsAppliedCount = - analysis.getRequiredBindings() - .stream() - .filter(c -> actualArrays.contains(c) && !analysis.getArrayBindings().contains(c)) - .count(); - - // unapplied multi-valued inputs result in multi-valued outputs, and unknowns are unknown - if (unknownIfArrays.size() > 0 || needsAppliedCount > 0) { - // always a multi-value string since wider engine does not yet support array types - return ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(ValueType.STRING); - } // if we got here, lets call it single value string output return new ColumnCapabilitiesImpl().setType(ValueType.STRING) .setHasMultipleValues(false) .setDictionaryEncoded(false); } + // fallback to return capabilities(columnName); } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/RowBasedExpressionColumnValueSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/RowBasedExpressionColumnValueSelector.java index 5a33bc771b4d..84117a4719da 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/RowBasedExpressionColumnValueSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/RowBasedExpressionColumnValueSelector.java @@ -45,17 +45,16 @@ public class RowBasedExpressionColumnValueSelector extends ExpressionColumnValue private final Int2ObjectMap transformedCache; public RowBasedExpressionColumnValueSelector( - Expr expression, - Expr.BindingAnalysis baseBindingAnalysis, - Expr.ObjectBinding bindings, - Set unknownColumnsSet + ExpressionPlan plan, + Expr.ObjectBinding bindings ) { - super(expression, bindings); - this.unknownColumns = unknownColumnsSet.stream() - .filter(x -> !baseBindingAnalysis.getArrayBindings().contains(x)) - .collect(Collectors.toList()); - this.baseBindingAnalysis = baseBindingAnalysis; + super(plan.getAppliedExpression(), bindings); + this.unknownColumns = plan.getUnknownInputs() + .stream() + .filter(x -> !plan.getAnalysis().getArrayBindings().contains(x)) + .collect(Collectors.toList()); + this.baseBindingAnalysis = plan.getAnalysis(); this.ignoredColumns = new HashSet<>(); this.transformedCache = new Int2ObjectArrayMap<>(unknownColumns.size()); } diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java index 84bf5fd7e741..b6f7733fc4b0 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -366,7 +366,7 @@ public ColumnCapabilities getColumnCapabilities(String column) SCALE_LIST_SELF_EXPLICIT.makeDimensionSelector(spec, factory); Assert.assertTrue(selectorImplicit instanceof SingleStringInputDimensionSelector); - Assert.assertTrue(selectorExplicit instanceof MultiValueExpressionDimensionSelector); + Assert.assertTrue(selectorExplicit instanceof ExpressionMultiValueDimensionSelector); } @Test From e89ad7d345e6d9dde232cba8bdf3e31eaf13ab56 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Thu, 17 Sep 2020 11:31:18 -0700 Subject: [PATCH 08/17] better names --- .../math/expr/BinaryLogicalOperatorExpr.java | 56 +++---- .../math/expr/BinaryMathOperatorExpr.java | 56 +++---- .../org/apache/druid/math/expr/Function.java | 143 +++++++++++++---- .../druid/math/expr/UnaryOperatorExpr.java | 8 +- .../BivariateFunctionVectorProcessor.java | 3 +- ...leOutDoubleInFunctionVectorProcessor.java} | 8 +- ...tDoubleLongInFunctionVectorProcessor.java} | 8 +- ...eOutDoublesInFunctionVectorProcessor.java} | 8 +- ...tLongDoubleInFunctionVectorProcessor.java} | 8 +- ...bleOutLongsInFunctionVectorProcessor.java} | 8 +- ...ngOutDoubleInFunctionVectorProcessor.java} | 8 +- ...LongOutLongInFunctionVectorProcessor.java} | 7 +- ...ongOutLongsInFunctionVectorProcessor.java} | 8 +- ...ngOutStringInFunctionVectorProcessor.java} | 8 +- ...ivariateFunctionVectorObjectProcessor.java | 4 + .../druid/math/expr/VectorExprSanityTest.java | 145 +++++++++--------- .../expression/TimestampFloorExprMacro.java | 4 +- .../druid/segment/virtual/ExpressionPlan.java | 10 -- 18 files changed, 305 insertions(+), 195 deletions(-) rename core/src/main/java/org/apache/druid/math/expr/vector/{DoubleUnivariateFunctionVectorProcessor.java => DoubleOutDoubleInFunctionVectorProcessor.java} (79%) rename core/src/main/java/org/apache/druid/math/expr/vector/{DoubleDoubleLongBivariateFunctionVectorProcessor.java => DoubleOutDoubleLongInFunctionVectorProcessor.java} (83%) rename core/src/main/java/org/apache/druid/math/expr/vector/{DoublesBivariateFunctionVectorProcessor.java => DoubleOutDoublesInFunctionVectorProcessor.java} (83%) rename core/src/main/java/org/apache/druid/math/expr/vector/{DoubleLongDoubleBivariateFunctionVectorProcessor.java => DoubleOutLongDoubleInFunctionVectorProcessor.java} (83%) rename core/src/main/java/org/apache/druid/math/expr/vector/{LongsDoubleBivariateFunctionVectorProcessor.java => DoubleOutLongsInFunctionVectorProcessor.java} (84%) rename core/src/main/java/org/apache/druid/math/expr/vector/{LongDoubleUnivariateFunctionVectorProcessor.java => LongOutDoubleInFunctionVectorProcessor.java} (79%) rename core/src/main/java/org/apache/druid/math/expr/vector/{LongUnivariateFunctionVectorProcessor.java => LongOutLongInFunctionVectorProcessor.java} (80%) rename core/src/main/java/org/apache/druid/math/expr/vector/{LongsBivariateFunctionVectorProcessor.java => LongOutLongsInFunctionVectorProcessor.java} (84%) rename core/src/main/java/org/apache/druid/math/expr/vector/{StringLongUnivariateFunctionVectorProcessor.java => LongOutStringInFunctionVectorProcessor.java} (77%) diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java index dc52389de956..5a298ae14105 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java @@ -21,10 +21,10 @@ import org.apache.druid.java.util.common.guava.Comparators; import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleDoubleLongBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleLongDoubleBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoublesBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongsBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoubleLongInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; @@ -90,7 +90,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -103,7 +103,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -118,7 +118,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -131,7 +131,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -211,7 +211,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -224,7 +224,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -239,7 +239,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -252,7 +252,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -331,7 +331,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -344,7 +344,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -359,7 +359,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -372,7 +372,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -452,7 +452,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -465,7 +465,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -480,7 +480,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -493,7 +493,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -572,7 +572,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -585,7 +585,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -600,7 +600,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -613,7 +613,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -692,7 +692,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -705,7 +705,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -720,7 +720,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -733,7 +733,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java index f56bdf033774..d4bade74b869 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java @@ -23,10 +23,10 @@ import com.google.common.primitives.Ints; import org.apache.druid.common.config.NullHandling; import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleDoubleLongBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleLongDoubleBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoublesBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongsBivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoubleLongInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; @@ -80,7 +80,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -93,7 +93,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -108,7 +108,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -121,7 +121,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -183,7 +183,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -196,7 +196,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -211,7 +211,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -224,7 +224,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -286,7 +286,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -299,7 +299,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -314,7 +314,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -327,7 +327,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -389,7 +389,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -402,7 +402,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -417,7 +417,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -430,7 +430,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -492,7 +492,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -505,7 +505,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -520,7 +520,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -533,7 +533,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -595,7 +595,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -608,7 +608,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -623,7 +623,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize @@ -636,7 +636,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 8ea71f788af5..90b9747bb177 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -28,13 +28,14 @@ import org.apache.druid.java.util.common.UOE; import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleDoubleLongBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleLongDoubleBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleUnivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoublesBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongDoubleUnivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongsBivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.StringLongUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoubleInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoubleLongInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutLongsInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutDoubleInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutStringInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -551,7 +552,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes VectorExprProcessor processor = null; if (args.size() == 1) { - processor = new StringLongUnivariateFunctionVectorProcessor( + processor = new LongOutStringInFunctionVectorProcessor( CastToTypeVectorProcessor.castToType(args.get(0).buildVectorized(inputTypes), ExprType.STRING), inputTypes.getMaxVectorSize() ) @@ -690,7 +691,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -702,7 +703,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -780,7 +781,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -792,7 +793,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -840,7 +841,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -852,7 +853,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -900,7 +901,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -912,7 +913,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1219,7 +1220,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1231,7 +1232,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1279,7 +1280,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1291,7 +1292,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1354,7 +1355,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1366,7 +1367,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1414,7 +1415,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes ExprType inputType = arg.getOutputType(inputTypes); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongDoubleUnivariateFunctionVectorProcessor( + processor = new LongOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1426,7 +1427,7 @@ public double apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleInFunctionVectorProcessor( arg.buildVectorized(inputTypes), inputTypes.getMaxVectorSize() ) @@ -1588,7 +1589,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1601,7 +1602,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1616,7 +1617,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1629,7 +1630,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1669,6 +1670,7 @@ protected ExprEval eval(double x, double y) { return ExprEval.of(Math.min(x, y)); } + @Override public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) { @@ -1685,7 +1687,7 @@ public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongsBivariateFunctionVectorProcessor( + processor = new LongOutLongsInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1698,7 +1700,7 @@ public long apply(long left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleLongDoubleBivariateFunctionVectorProcessor( + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1713,7 +1715,7 @@ public double apply(long left, double right) } } else if (ExprType.DOUBLE.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new DoubleDoubleLongBivariateFunctionVectorProcessor( + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1726,7 +1728,7 @@ public double apply(double left, long right) } }; } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoublesBivariateFunctionVectorProcessor( + processor = new DoubleOutDoublesInFunctionVectorProcessor( args.get(0).buildVectorized(inputTypes), args.get(1).buildVectorized(inputTypes), maxVectorSize @@ -1775,6 +1777,83 @@ protected ExprEval eval(double x, double y) { return ExprEval.of(Math.pow(x, y)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + final ExprType leftType = args.get(0).getOutputType(inputTypes); + final ExprType rightType = args.get(1).getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutLongsInFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, long right) + { + return Math.pow(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.pow(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.pow(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + args.get(0).buildVectorized(inputTypes), + args.get(1).buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.pow(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(this); + } + return (VectorExprProcessor) processor; + } } class Scalb extends BivariateFunction diff --git a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java index 813eec85c887..79d5b995006d 100644 --- a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java @@ -23,8 +23,8 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.DoubleUnivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleOutDoubleInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutLongInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; @@ -156,7 +156,7 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputT final int maxVectorSize = inputTypes.getMaxVectorSize(); VectorExprProcessor processor = null; if (ExprType.LONG.equals(inputType)) { - processor = new LongUnivariateFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) + processor = new LongOutLongInFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) { @Override public long apply(long input) @@ -165,7 +165,7 @@ public long apply(long input) } }; } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleUnivariateFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) + processor = new DoubleOutDoubleInFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) { @Override public double apply(double input) diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java index 52bb59b020fb..4e3cc6b76bde 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java @@ -26,7 +26,8 @@ * output, and are backed by a primitive values instead of an object values (and need to use the null vectors instead of * checking the vector themselves for nulls) */ -public abstract class BivariateFunctionVectorProcessor implements VectorExprProcessor +public abstract class BivariateFunctionVectorProcessor + implements VectorExprProcessor { final VectorExprProcessor left; final VectorExprProcessor right; diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java similarity index 79% rename from core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java index 415462017367..1fb4edab3e99 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class DoubleUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorProcessor +/** + * specialized {@link UnivariateFunctionVectorProcessor} for processing (double[]) -> double[] + */ +public abstract class DoubleOutDoubleInFunctionVectorProcessor + extends UnivariateFunctionVectorProcessor { - public DoubleUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public DoubleOutDoubleInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new double[maxVectorSize]); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java similarity index 83% rename from core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java index 65ecde018a3e..e608a77b5340 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleDoubleLongBivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class DoubleDoubleLongBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +/** + * specialized {@link BivariateFunctionVectorProcessor} for processing (double[], long[]) -> double[] + */ +public abstract class DoubleOutDoubleLongInFunctionVectorProcessor + extends BivariateFunctionVectorProcessor { - public DoubleDoubleLongBivariateFunctionVectorProcessor( + public DoubleOutDoubleLongInFunctionVectorProcessor( VectorExprProcessor left, VectorExprProcessor right, int maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java similarity index 83% rename from core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java index bd17a8f1d410..7f9d42ced6d2 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoublesBivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class DoublesBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +/** + * specialized {@link BivariateFunctionVectorProcessor} for processing (double[], double[]) -> double[] + */ +public abstract class DoubleOutDoublesInFunctionVectorProcessor + extends BivariateFunctionVectorProcessor { - public DoublesBivariateFunctionVectorProcessor( + public DoubleOutDoublesInFunctionVectorProcessor( VectorExprProcessor left, VectorExprProcessor right, int maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java similarity index 83% rename from core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java index f828a673d24a..577b39d9efad 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleLongDoubleBivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class DoubleLongDoubleBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +/** + * specialized {@link BivariateFunctionVectorProcessor} for processing (long[], double[]) -> double[] + */ +public abstract class DoubleOutLongDoubleInFunctionVectorProcessor + extends BivariateFunctionVectorProcessor { - public DoubleLongDoubleBivariateFunctionVectorProcessor( + public DoubleOutLongDoubleInFunctionVectorProcessor( VectorExprProcessor left, VectorExprProcessor right, int maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java similarity index 84% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java index b50dc379906e..1cb3e9516edc 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongsDoubleBivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class LongsDoubleBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +/** + * specialized {@link BivariateFunctionVectorProcessor} for processing (long[], long[]) -> double[] + */ +public abstract class DoubleOutLongsInFunctionVectorProcessor + extends BivariateFunctionVectorProcessor { - public LongsDoubleBivariateFunctionVectorProcessor( + public DoubleOutLongsInFunctionVectorProcessor( VectorExprProcessor left, VectorExprProcessor right, int maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorProcessor.java similarity index 79% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorProcessor.java index 292444313b89..d8045b5855dc 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongDoubleUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class LongDoubleUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorProcessor +/** + * specialized {@link UnivariateFunctionVectorProcessor} for processing (long[]) -> double[] + */ +public abstract class LongOutDoubleInFunctionVectorProcessor + extends UnivariateFunctionVectorProcessor { - public LongDoubleUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public LongOutDoubleInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new double[maxVectorSize]); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java similarity index 80% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java index 15ea8f0abee4..e4ad864a87fb 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java @@ -21,9 +21,12 @@ import org.apache.druid.math.expr.ExprType; -public abstract class LongUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorProcessor +/** + * specialized {@link UnivariateFunctionVectorProcessor} for processing (long[]) -> long[] + */ +public abstract class LongOutLongInFunctionVectorProcessor extends UnivariateFunctionVectorProcessor { - public LongUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public LongOutLongInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new long[maxVectorSize]); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java similarity index 84% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java index 9b294d6d1f50..cc26ebd47e7b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongsBivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class LongsBivariateFunctionVectorProcessor extends BivariateFunctionVectorProcessor +/** + * specialized {@link BivariateFunctionVectorProcessor} for processing (long[], long[]) -> long[] + */ +public abstract class LongOutLongsInFunctionVectorProcessor + extends BivariateFunctionVectorProcessor { - public LongsBivariateFunctionVectorProcessor( + public LongOutLongsInFunctionVectorProcessor( VectorExprProcessor left, VectorExprProcessor right, int maxVectorSize diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java similarity index 77% rename from core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java index 512324c5a82c..0537da260d07 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/StringLongUnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java @@ -21,9 +21,13 @@ import org.apache.druid.math.expr.ExprType; -public abstract class StringLongUnivariateFunctionVectorProcessor extends UnivariateFunctionVectorObjectProcessor +/** + * specialized {@link UnivariateFunctionVectorObjectProcessor} for processing (String[]) -> long[] + */ +public abstract class LongOutStringInFunctionVectorProcessor + extends UnivariateFunctionVectorObjectProcessor { - public StringLongUnivariateFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public LongOutStringInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new long[maxVectorSize]); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java index 2b37f86d3f29..6679c177efd0 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java @@ -21,6 +21,10 @@ import org.apache.druid.math.expr.Expr; +/** + * common machinery for processing single input operators and functions, which are backed by an object value instead of + * a primitive value (so do not need to use the null vector, and instead can check the value vector itself for nulls) + */ public abstract class UnivariateFunctionVectorObjectProcessor implements VectorExprProcessor { final VectorExprProcessor processor; diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index dfd6c613adc8..d2b117a1691c 100644 --- a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -60,6 +60,79 @@ public class VectorExprSanityTest extends InitializedNullHandlingTest .put("s2", ExprType.STRING) .build(); + @Test + public void testUnaryOperators() + { + final String[] functions = new String[]{"-"}; + final String[] templates = new String[]{"%sd1", "%sl1"}; + + testFunctions(types, templates, functions); + } + + @Test + public void testBinaryOperators() + { + final String[] columns = new String[]{"d1", "d2", "l1", "l2", "1", "1.0"}; + final String[][] templateInputs = makeTemplateArgs(columns, columns); + final String[] templates = + Arrays.stream(templateInputs) + .map(i -> StringUtils.format("%s %s %s", i[0], "%s", i[1])) + .toArray(String[]::new); + final String[] args = new String[]{"+", "-", "*", "/", "^", "%", ">", ">=", "<", "<=", "==", "!="}; + + testFunctions(types, templates, args); + } + + @Test + public void testBinaryOperatorTrees() + { + final String[] columns = new String[]{"d1", "l1", "1", "1.0"}; + final String[] columns2 = new String[]{"d2", "l2", "2", "2.0"}; + final String[][] templateInputs = makeTemplateArgs(columns, columns2, columns); + final String[] templates = + Arrays.stream(templateInputs) + .map(i -> StringUtils.format("(%s %s %s) %s %s", i[0], "%s", i[1], "%s", i[2])) + .toArray(String[]::new); + final String[] ops = new String[]{"+", "-", "*", "/"}; + final String[][] args = makeTemplateArgs(ops, ops); + testFunctions(types, templates, args); + } + + @Test + public void testUnivariateFunctions() + { + final String[] functions = new String[]{"parse_long"}; + final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testUnivariateMathFunctions() + { + final String[] functions = new String[]{"atan", "cos", "cosh", "cot", "sin", "sinh", "tan", "tanh"}; + final String[] templates = new String[]{"%s(l1)", "%s(d1)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testBivariateMathFunctions() + { + final String[] functions = new String[]{"max", "min", "pow"}; + final String[] templates = new String[]{"%s(d1, d2)", "%s(d1, l1)", "%s(l1, d1)", "%s(l1, l2)"}; + testFunctions(types, templates, functions); + } + + @Test + public void testCast() + { + final String[] columns = new String[]{"d1", "l1", "s1"}; + final String[] castTo = new String[]{"'STRING'", "'LONG'", "'DOUBLE'"}; + final String[][] args = makeTemplateArgs(columns, castTo); + final String[] templates = new String[]{"cast(%s, %s)"}; + testFunctions(types, templates, args); + } + + static void testFunctions(Map types, String[] templates, String[] args) { for (String template : templates) { @@ -260,78 +333,6 @@ static String[][] makeTemplateArgs(String[] arg1, String[] arg2, String[] arg3) .toArray(String[][]::new); } - @Test - public void testUnaryOperators() - { - final String[] functions = new String[]{"-"}; - final String[] templates = new String[]{"%sd1", "%sl1"}; - - testFunctions(types, templates, functions); - } - - @Test - public void testBinaryOperators() - { - final String[] columns = new String[]{"d1", "d2", "l1", "l2", "1", "1.0"}; - final String[][] templateInputs = makeTemplateArgs(columns, columns); - final String[] templates = - Arrays.stream(templateInputs) - .map(i -> StringUtils.format("%s %s %s", i[0], "%s", i[1])) - .toArray(String[]::new); - final String[] args = new String[]{"+", "-", "*", "/", "^", "%", ">", ">=", "<", "<=", "==", "!="}; - - testFunctions(types, templates, args); - } - - @Test - public void testBinaryOperatorTrees() - { - final String[] columns = new String[]{"d1", "l1", "1", "1.0"}; - final String[] columns2 = new String[]{"d2", "l2", "2", "2.0"}; - final String[][] templateInputs = makeTemplateArgs(columns, columns2, columns); - final String[] templates = - Arrays.stream(templateInputs) - .map(i -> StringUtils.format("(%s %s %s) %s %s", i[0], "%s", i[1], "%s", i[2])) - .toArray(String[]::new); - final String[] ops = new String[]{"+", "-", "*", "/"}; - final String[][] args = makeTemplateArgs(ops, ops); - testFunctions(types, templates, args); - } - - @Test - public void testUnivariateFunctions() - { - final String[] functions = new String[]{"parse_long"}; - final String[] templates = new String[]{"%s(s1)", "%s(l1)", "%s(d1)"}; - testFunctions(types, templates, functions); - } - - @Test - public void testUnivariateMathFunctions() - { - final String[] functions = new String[]{"atan", "cos", "cosh", "cot", "sin", "sinh", "tan", "tanh"}; - final String[] templates = new String[]{"%s(l1)", "%s(d1)"}; - testFunctions(types, templates, functions); - } - - @Test - public void testBivariateMathFunctions() - { - final String[] functions = new String[]{"max", "min"}; - final String[] templates = new String[]{"%s(d1, d2)", "%s(d1, l1)", "%s(l1, d1)", "%s(l1, l2)"}; - testFunctions(types, templates, functions); - } - - @Test - public void testCast() - { - final String[] columns = new String[]{"d1", "l1", "s1"}; - final String[] castTo = new String[]{"'STRING'", "'LONG'", "'DOUBLE'"}; - final String[][] args = makeTemplateArgs(columns, castTo); - final String[] templates = new String[]{"cast(%s, %s)"}; - testFunctions(types, templates, args); - } - static class SettableObjectBinding implements Expr.ObjectBinding { private final Map bindings; diff --git a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java index d5bd8eea72d9..c1555d21e913 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java @@ -27,7 +27,7 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; -import org.apache.druid.math.expr.vector.LongUnivariateFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.LongOutLongInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nonnull; @@ -133,7 +133,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { VectorExprProcessor processor; - processor = new LongUnivariateFunctionVectorProcessor( + processor = new LongOutLongInFunctionVectorProcessor( CastToTypeVectorProcessor.castToType(args.get(0).buildVectorized(inputTypes), ExprType.LONG), inputTypes.getMaxVectorSize() ) diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java index 04481b7e5c77..07e6ce3aa984 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionPlan.java @@ -128,11 +128,6 @@ public boolean any(Trait... flags) return any(traits, flags); } - public boolean none(Trait... flags) - { - return none(traits, flags); - } - @Nullable public ExprType getOutputType() { @@ -155,11 +150,6 @@ public Set getUnknownInputs() return unknownInputs; } - public List getUnappliedInputs() - { - return unappliedInputs; - } - static boolean is(EnumSet traits, Trait... args) { return Arrays.stream(args).allMatch(traits::contains); From 482192457ec02e86ef4806f0f8b5d6118d473bbf Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 18 Sep 2020 03:39:57 -0700 Subject: [PATCH 09/17] remove unused method, add pi --- .../org/apache/druid/math/expr/Function.java | 32 +++++++++++++++++++ .../BivariateFunctionVectorProcessor.java | 2 -- ...utDoubleLongInFunctionVectorProcessor.java | 6 ---- ...leOutDoublesInFunctionVectorProcessor.java | 6 ---- ...utLongDoubleInFunctionVectorProcessor.java | 6 ---- ...ubleOutLongsInFunctionVectorProcessor.java | 6 ---- ...LongOutLongsInFunctionVectorProcessor.java | 6 ---- .../druid/math/expr/VectorExprSanityTest.java | 3 +- 8 files changed, 33 insertions(+), 34 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 90b9747bb177..2c42d0c9fb0b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -33,9 +33,11 @@ import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.DoubleOutLongsInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.DoubleVectorExprEval; import org.apache.druid.math.expr.vector.LongOutDoubleInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.LongOutStringInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorExprEval; import org.apache.druid.math.expr.vector.VectorExprProcessor; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -611,6 +613,36 @@ public ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args { return ExprType.DOUBLE; } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return true; + } + + @Override + public VectorExprProcessor asVectorProcessor( + Expr.VectorInputBindingTypes inputTypes, List args + ) + { + final double[] pi = new double[inputTypes.getMaxVectorSize()]; + Arrays.fill(pi, PI); + final DoubleVectorExprEval eval = new DoubleVectorExprEval(pi,null); + return new VectorExprProcessor() + { + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + return (VectorExprEval) eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + }; + } } class Abs extends UnivariateMathFunction diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java index 4e3cc6b76bde..41f8376b8e52 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java @@ -83,7 +83,5 @@ public final VectorExprEval evalVector(Expr.VectorInputBinding bindings abstract void processIndex(TLeftInput leftInput, TRightInput rightInput, int i); - abstract void processNull(int i); - abstract VectorExprEval asEval(); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java index e608a77b5340..6a14607d3a88 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java @@ -50,12 +50,6 @@ final void processIndex(double[] leftInput, long[] rightInput, int i) outValues[i] = apply(leftInput[i], rightInput[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java index 7f9d42ced6d2..6a9f194cb2b5 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java @@ -50,12 +50,6 @@ final void processIndex(double[] leftInput, double[] rightInput, int i) outValues[i] = apply(leftInput[i], rightInput[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java index 577b39d9efad..a66e8bbfc38c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java @@ -50,12 +50,6 @@ final void processIndex(long[] leftInput, double[] rightInput, int i) outValues[i] = apply(leftInput[i], rightInput[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java index 1cb3e9516edc..26425f1b5d35 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java @@ -50,12 +50,6 @@ final void processIndex(long[] leftInput, long[] rightInput, int i) outValues[i] = apply(leftInput[i], rightInput[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java index cc26ebd47e7b..788668a487a8 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java @@ -50,12 +50,6 @@ final void processIndex(long[] leftInput, long[] rightInput, int i) outValues[i] = apply(leftInput[i], rightInput[i]); } - @Override - void processNull(int i) - { - outValues[i] = 0; - } - @Override final VectorExprEval asEval() { diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index d2b117a1691c..ea250322ed3e 100644 --- a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -110,7 +110,7 @@ public void testUnivariateFunctions() public void testUnivariateMathFunctions() { final String[] functions = new String[]{"atan", "cos", "cosh", "cot", "sin", "sinh", "tan", "tanh"}; - final String[] templates = new String[]{"%s(l1)", "%s(d1)"}; + final String[] templates = new String[]{"%s(l1)", "%s(d1)", "%s(pi())"}; testFunctions(types, templates, functions); } @@ -132,7 +132,6 @@ public void testCast() testFunctions(types, templates, args); } - static void testFunctions(Map types, String[] templates, String[] args) { for (String template : templates) { From 9698124c0b4bdb621b0fe23f08426c5122910afe Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 18 Sep 2020 15:14:31 -0700 Subject: [PATCH 10/17] move vector processor builders into static methods --- .../query/SqlExpressionBenchmark.java | 48 +- .../math/expr/BinaryLogicalOperatorExpr.java | 408 +------- .../math/expr/BinaryMathOperatorExpr.java | 408 +------- .../apache/druid/math/expr/ConstantExpr.java | 112 +-- .../org/apache/druid/math/expr/Function.java | 563 +---------- .../druid/math/expr/IdentifierExpr.java | 19 +- .../druid/math/expr/UnaryOperatorExpr.java | 34 +- .../vector/VectorComparisonProcessors.java | 477 +++++++++ .../expr/vector/VectorMathProcessors.java | 947 ++++++++++++++++++ .../math/expr/vector/VectorProcessors.java | 131 +++ .../segment/RowBasedStorageAdapterTest.java | 2 +- 11 files changed, 1647 insertions(+), 1502 deletions(-) create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java create mode 100644 core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 0a098783688c..a4549367bdab 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -190,33 +190,33 @@ public String getFormatString() @Param({ // non-expression reference - "0", +// "0", "1", - "2", - "3", - "4", - "5", - "6", +// "2", +// "3", +// "4", +// "5", +// "6", // expressions "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "20", - "21", - "22", - "23", - "24", - "25" +// "8", +// "9", +// "10", +// "11", +// "12", +// "13", +// "14", +// "15", +// "16", +// "17", +// "18", +// "19", +// "20", +// "21", +// "22", +// "23", +// "24", +// "25" }) private String query; diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java index 5a298ae14105..b207ed1fe972 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java @@ -20,11 +20,7 @@ package org.apache.druid.math.expr; import org.apache.druid.java.util.common.guava.Comparators; -import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoubleLongInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; +import org.apache.druid.math.expr.vector.VectorComparisonProcessors; import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; @@ -83,72 +79,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left < right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) < 0); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) < 0); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) < 0); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorComparisonProcessors.lessThan(inputTypes, left, right); } } @@ -204,72 +135,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left <= right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) <= 0); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) <= 0); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) <= 0); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorComparisonProcessors.lessThanOrEqual(inputTypes, left, right); } } @@ -324,72 +190,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left > right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) > 0); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) > 0); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) > 0); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorComparisonProcessors.greaterThan(inputTypes, left, right); } } @@ -445,72 +246,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left >= right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(Double.compare(left, right) >= 0); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(Double.compare(left, right) >= 0); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(Double.compare(left, right) >= 0); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorComparisonProcessors.greaterThanOrEqual(inputTypes, left, right); } } @@ -565,72 +301,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left == right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(left == right); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(left == right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(left == right); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorComparisonProcessors.equal(inputTypes, left, right); } } @@ -685,72 +356,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Evals.asLong(left != right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Evals.asDouble(left != right); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Evals.asDouble(left != right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Evals.asDouble(left != right); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorComparisonProcessors.notEqual(inputTypes, left, right); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java index d4bade74b869..a0d496640e0d 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java @@ -22,12 +22,8 @@ import com.google.common.math.LongMath; import com.google.common.primitives.Ints; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoubleLongInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.VectorMathProcessors; import javax.annotation.Nullable; @@ -73,72 +69,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return left + right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return (double) left + right; - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return left + (double) right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return left + right; - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.plus(inputTypes, left, right); } } @@ -176,72 +107,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return left - right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return (double) left - right; - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return left - (double) right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return left - right; - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.minus(inputTypes, left, right); } } @@ -279,72 +145,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return left * right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return (double) left * right; - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return left * (double) right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return left * right; - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.multiply(inputTypes, left, right); } } @@ -382,72 +183,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return left / right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return (double) left / right; - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return left / (double) right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return left / right; - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.divide(inputTypes, left, right); } } @@ -485,72 +221,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return LongMath.pow(left, Ints.checkedCast(right)); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Math.pow(left, right); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Math.pow(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Math.pow(left, right); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.power(inputTypes, left, right); } } @@ -588,71 +259,6 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return left % right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return (double) left % right; - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return left % (double) right; - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - left.buildVectorized(inputTypes), - right.buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return left % right; - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.modulo(inputTypes, left, right); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java b/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java index 7a7b80f11e7b..c8c7f2b3909a 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java @@ -23,11 +23,8 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.DoubleVectorExprEval; -import org.apache.druid.math.expr.vector.LongVectorExprEval; -import org.apache.druid.math.expr.vector.StringVectorExprEval; -import org.apache.druid.math.expr.vector.VectorExprEval; import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.VectorProcessors; import javax.annotation.Nullable; import java.util.Arrays; @@ -147,29 +144,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final long[] constant = new long[inputTypes.getMaxVectorSize()]; - final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; - if (value != null) { - Arrays.fill(constant, value); - } else { - Arrays.fill(nulls, true); - } - return new VectorExprProcessor() - { - private final VectorExprEval eval = - (VectorExprEval) new LongVectorExprEval(constant, nulls); - @Override - public VectorExprEval evalVector(VectorInputBinding bindings) - { - return eval; - } - - @Override - public ExprType getOutputType() - { - return ExprType.LONG; - } - }; + return VectorProcessors.constantLong(value, inputTypes.getMaxVectorSize()); } @Override @@ -214,25 +189,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final long[] constant = new long[inputTypes.getMaxVectorSize()]; - final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; - Arrays.fill(nulls, true); - return new VectorExprProcessor() - { - private final VectorExprEval eval = - (VectorExprEval) new LongVectorExprEval(constant, nulls); - @Override - public VectorExprEval evalVector(VectorInputBinding bindings) - { - return eval; - } - - @Override - public ExprType getOutputType() - { - return ExprType.LONG; - } - }; + return VectorProcessors.constantLong(null, inputTypes.getMaxVectorSize()); } @Override @@ -342,29 +299,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final double[] constant = new double[inputTypes.getMaxVectorSize()]; - final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; - if (value != null) { - Arrays.fill(constant, value); - } else { - Arrays.fill(nulls, true); - } - return new VectorExprProcessor() - { - private final VectorExprEval eval = - (VectorExprEval) new DoubleVectorExprEval(constant, nulls); - @Override - public VectorExprEval evalVector(VectorInputBinding bindings) - { - return eval; - } - - @Override - public ExprType getOutputType() - { - return ExprType.DOUBLE; - } - }; + return VectorProcessors.constantDouble(value, inputTypes.getMaxVectorSize()); } @Override public boolean equals(Object o) @@ -408,25 +343,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final double[] constant = new double[inputTypes.getMaxVectorSize()]; - final boolean[] nulls = new boolean[inputTypes.getMaxVectorSize()]; - Arrays.fill(nulls, true); - return new VectorExprProcessor() - { - private final VectorExprEval eval = - (VectorExprEval) new DoubleVectorExprEval(constant, nulls); - @Override - public VectorExprEval evalVector(VectorInputBinding bindings) - { - return eval; - } - - @Override - public ExprType getOutputType() - { - return ExprType.DOUBLE; - } - }; + return VectorProcessors.constantDouble(null, inputTypes.getMaxVectorSize()); } @Override @@ -538,24 +455,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final String[] strings = new String[inputTypes.getMaxVectorSize()]; - Arrays.fill(strings, value); - return new VectorExprProcessor() - { - private final VectorExprEval eval = (VectorExprEval) new StringVectorExprEval(strings); - - @Override - public VectorExprEval evalVector(VectorInputBinding bindings) - { - return eval; - } - - @Override - public ExprType getOutputType() - { - return ExprType.STRING; - } - }; + return VectorProcessors.constantString(value, inputTypes.getMaxVectorSize()); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index 2c42d0c9fb0b..c238dc6e224f 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -26,19 +26,10 @@ import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.UOE; -import org.apache.druid.math.expr.vector.BivariateFunctionVectorProcessor; import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoubleInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoubleLongInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutDoublesInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutLongDoubleInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleOutLongsInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.DoubleVectorExprEval; -import org.apache.druid.math.expr.vector.LongOutDoubleInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongOutLongsInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongOutStringInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.VectorExprEval; import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.VectorMathProcessors; +import org.apache.druid.math.expr.vector.VectorProcessors; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -545,41 +536,18 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) @Override public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) { - return args.size() == 1 && inputTypes.canVectorize(args); + return (args.size() == 1 || args.get(1).isLiteral()) && inputTypes.canVectorize(args); } @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - VectorExprProcessor processor = null; - - if (args.size() == 1) { - processor = new LongOutStringInFunctionVectorProcessor( - CastToTypeVectorProcessor.castToType(args.get(0).buildVectorized(inputTypes), ExprType.STRING), - inputTypes.getMaxVectorSize() - ) - { - @Override - public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) - { - try { - longs[i] = Long.parseLong(strings[i], 10); - outputNulls[i] = false; - } - catch (NumberFormatException e) { - longs[i] = 0L; - outputNulls[i] = NullHandling.sqlCompatible(); - } - } - }; - } - - if (processor == null) { - // not yet implemented, how did we get here - throw Exprs.cannotVectorize(this); + if (args.size() == 1 || args.get(1).isLiteral()) { + final int radix = args.size() == 1 ? 10 : ((Number) args.get(1).getLiteralValue()).intValue(); + return VectorProcessors.parseLong(inputTypes, args.get(0), radix); } - - return (VectorExprProcessor) processor; + // not yet implemented, how did we get here + throw Exprs.cannotVectorize(this); } } @@ -621,27 +589,9 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor( - Expr.VectorInputBindingTypes inputTypes, List args - ) + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - final double[] pi = new double[inputTypes.getMaxVectorSize()]; - Arrays.fill(pi, PI); - final DoubleVectorExprEval eval = new DoubleVectorExprEval(pi,null); - return new VectorExprProcessor() - { - @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) - { - return (VectorExprEval) eval; - } - - @Override - public ExprType getOutputType() - { - return ExprType.DOUBLE; - } - }; + return VectorProcessors.constantDouble(PI, inputTypes.getMaxVectorSize()); } } @@ -719,40 +669,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.atan(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.atan(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.atan(inputTypes, args.get(0)); } } @@ -809,40 +726,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.cos(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.cos(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.cos(inputTypes, args.get(0)); } } @@ -869,40 +753,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.cosh(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.cosh(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.cosh(inputTypes, args.get(0)); } } @@ -929,40 +780,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.cos(input) / Math.sin(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.cos(input) / Math.sin(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.cot(inputTypes, args.get(0)); } } @@ -985,6 +803,18 @@ protected ExprEval eval(final double x, final double y) { return ExprEval.of((long) (x / y)); } + + @Override + public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) + { + return inputTypes.areNumeric(args) && inputTypes.canVectorize(args); + } + + @Override + public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + { + return VectorMathProcessors.divide(inputTypes, args.get(0), args.get(1)); + } } class Exp extends DoubleUnivariateMathFunction @@ -1248,40 +1078,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.sin(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.sin(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.sin(inputTypes, args.get(0)); } } @@ -1308,40 +1105,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.sinh(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.sinh(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.sinh(inputTypes, args.get(0)); } } @@ -1383,40 +1147,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.tan(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.tan(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.tan(inputTypes, args.get(0)); } } @@ -1443,40 +1174,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - Expr arg = args.get(0); - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.tanh(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.tanh(input); - } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - - return (VectorExprProcessor) processor; + return VectorMathProcessors.tanh(inputTypes, args.get(0)); } } @@ -1614,72 +1312,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - final ExprType leftType = args.get(0).getOutputType(inputTypes); - final ExprType rightType = args.get(1).getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Math.max(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Math.max(left, right); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Math.max(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Math.max(left, right); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.max(inputTypes, args.get(0), args.get(1)); } } @@ -1712,72 +1345,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - final ExprType leftType = args.get(0).getOutputType(inputTypes); - final ExprType rightType = args.get(1).getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public long apply(long left, long right) - { - return Math.min(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Math.min(left, right); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Math.min(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Math.min(left, right); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.min(inputTypes, args.get(0), args.get(1)); } } @@ -1819,72 +1387,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) @Override public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { - final ExprType leftType = args.get(0).getOutputType(inputTypes); - final ExprType rightType = args.get(1).getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutLongsInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, long right) - { - return Math.pow(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(long left, double right) - { - return Math.pow(left, right); - } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, long right) - { - return Math.pow(left, right); - } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( - args.get(0).buildVectorized(inputTypes), - args.get(1).buildVectorized(inputTypes), - maxVectorSize - ) - { - @Override - public double apply(double left, double right) - { - return Math.pow(left, right); - } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.doublePower(inputTypes, args.get(0), args.get(1)); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java index d9faab0a43f5..d2a8cc2ae382 100644 --- a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java @@ -118,12 +118,6 @@ public ExprType getOutputType(InputBindingTypes inputTypes) return inputTypes.getType(binding); } - @Override - public boolean canVectorize(InputBindingTypes inputTypes) - { - return true; - } - @Override public ExprEval eval(ObjectBinding bindings) { @@ -149,10 +143,21 @@ public Expr visit(Shuttle shuttle) return shuttle.visit(this); } + @Override + public boolean canVectorize(InputBindingTypes inputTypes) + { + return inputTypes.getType(binding) != null; + } + @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - switch (inputTypes.getType(binding)) { + ExprType inputType = inputTypes.getType(binding); + + if (inputType == null) { + throw Exprs.cannotVectorize(this); + } + switch (inputType) { case LONG: return new IdentifierVectorProcessor(ExprType.LONG) { diff --git a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java index 79d5b995006d..05e1b8a4bbff 100644 --- a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java @@ -23,9 +23,8 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.DoubleOutDoubleInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.LongOutLongInFunctionVectorProcessor; import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.VectorMathProcessors; import javax.annotation.Nullable; import java.util.Objects; @@ -148,36 +147,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) @Override public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - final ExprType inputType = expr.getOutputType(inputTypes); - if (inputType == null) { - throw Exprs.cannotVectorize(this); - } - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutLongInFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) - { - @Override - public long apply(long input) - { - return -input; - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor(expr.buildVectorized(inputTypes), maxVectorSize) - { - @Override - public double apply(double input) - { - return -input; - } - }; - } - if (processor == null) { - throw Exprs.cannotVectorize(this); - } - return (VectorExprProcessor) processor; + return VectorMathProcessors.negate(inputTypes, expr); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java new file mode 100644 index 000000000000..c82958880d86 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java @@ -0,0 +1,477 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.math.expr.Evals; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Exprs; + +public class VectorComparisonProcessors +{ + private VectorComparisonProcessors() + { + // No instantiation + } + + public static VectorExprProcessor equal( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right + ) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left == right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(left == right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(left == right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(left == right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor notEqual( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right + ) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left != right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(left != right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(left != right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(left != right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor greaterThanOrEqual( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right + ) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left >= right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) >= 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) >= 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) >= 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor greaterThan( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right + ) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left > right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) > 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) > 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) > 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor lessThanOrEqual( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right + ) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left <= right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) <= 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) <= 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) <= 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor lessThan( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right + ) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Evals.asLong(left < right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Evals.asDouble(Double.compare(left, right) < 0); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Evals.asDouble(Double.compare(left, right) < 0); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Evals.asDouble(Double.compare(left, right) < 0); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java new file mode 100644 index 000000000000..95c73f17dc6c --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java @@ -0,0 +1,947 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import com.google.common.math.LongMath; +import com.google.common.primitives.Ints; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Exprs; + +public class VectorMathProcessors +{ + public static VectorExprProcessor plus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left + right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left + right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left + (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left + right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor minus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left - right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left - right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left - (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left - right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor multiply(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left * right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left * right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left * (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left * right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor divide(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left / right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left / right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left / (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left / right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor modulo(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return left % right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return (double) left % right; + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return left % (double) right; + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return left % right; + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor negate(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + final ExprType inputType = arg.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutLongInFunctionVectorProcessor(arg.buildVectorized(inputTypes), maxVectorSize) + { + @Override + public long apply(long input) + { + return -input; + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor(arg.buildVectorized(inputTypes), maxVectorSize) + { + @Override + public double apply(double input) + { + return -input; + } + }; + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor power(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return LongMath.pow(left, Ints.checkedCast(right)); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.pow(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.pow(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.pow(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor doublePower(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(left.getOutputType(inputTypes))) { + if (ExprType.LONG.equals(right.getOutputType(inputTypes))) { + processor = new DoubleOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long left, long right) + { + return Math.pow(left, right); + } + }; + } + } + + if (processor != null) { + return (VectorExprProcessor) processor; + } + return power(inputTypes, left, right); + } + + public static VectorExprProcessor max(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Math.max(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.max(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.max(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.max(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor min(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + final ExprType leftType = left.getOutputType(inputTypes); + final ExprType rightType = right.getOutputType(inputTypes); + + final int maxVectorSize = inputTypes.getMaxVectorSize(); + BivariateFunctionVectorProcessor processor = null; + if (ExprType.LONG.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new LongOutLongsInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public long apply(long left, long right) + { + return Math.min(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(long left, double right) + { + return Math.min(left, right); + } + }; + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, long right) + { + return Math.min(left, right); + } + }; + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = new DoubleOutDoublesInFunctionVectorProcessor( + left.buildVectorized(inputTypes), + right.buildVectorized(inputTypes), + maxVectorSize + ) + { + @Override + public double apply(double left, double right) + { + return Math.min(left, right); + } + }; + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor atan(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.atan(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.atan(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor cos(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cos(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cos(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor cosh(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cosh(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cosh(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor cot(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cos(input) / Math.sin(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cos(input) / Math.sin(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor sin(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.sin(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.sin(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor sinh(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.sinh(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.sinh(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor tan(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.tan(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.tan(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor tanh(Expr.VectorInputBindingTypes inputTypes, Expr arg) + { + ExprType inputType = arg.getOutputType(inputTypes); + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = new LongOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.tanh(input); + } + }; + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.tanh(input); + } + }; + } + + if (processor == null) { + throw Exprs.cannotVectorize(); + } + + return (VectorExprProcessor) processor; + } + + private VectorMathProcessors() + { + // No instantiation + } +} diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java new file mode 100644 index 000000000000..d8c873272310 --- /dev/null +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.math.expr.vector; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprType; + +import javax.annotation.Nullable; +import java.util.Arrays; + +public class VectorProcessors +{ + public static VectorExprProcessor constantString(@Nullable String constant, int maxVectorSize) + { + final String[] strings = new String[maxVectorSize]; + Arrays.fill(strings, constant); + final StringVectorExprEval eval = new StringVectorExprEval(strings); + return new VectorExprProcessor() + { + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + return (VectorExprEval) eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.STRING; + } + }; + } + + public static VectorExprProcessor constantDouble(@Nullable Double constant, int maxVectorSize) + { + final double[] doubles = new double[maxVectorSize]; + final boolean[] nulls; + if (constant == null) { + nulls = new boolean[maxVectorSize]; + Arrays.fill(nulls, true); + } else { + nulls = null; + Arrays.fill(doubles, constant); + } + final DoubleVectorExprEval eval = new DoubleVectorExprEval(doubles, nulls); + return new VectorExprProcessor() + { + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + return (VectorExprEval) eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.DOUBLE; + } + }; + } + + public static VectorExprProcessor constantLong(@Nullable Long constant, int maxVectorSize) + { + final long[] longs = new long[maxVectorSize]; + final boolean[] nulls; + if (constant == null) { + nulls = new boolean[maxVectorSize]; + Arrays.fill(nulls, true); + } else { + nulls = null; + Arrays.fill(longs, constant); + } + final LongVectorExprEval eval = new LongVectorExprEval(longs, nulls); + return new VectorExprProcessor() + { + @Override + public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + { + return (VectorExprEval) eval; + } + + @Override + public ExprType getOutputType() + { + return ExprType.LONG; + } + }; + } + + public static VectorExprProcessor parseLong(Expr.VectorInputBindingTypes inputTypes, Expr arg, int radix) + { + final VectorExprProcessor processor = new LongOutStringInFunctionVectorProcessor( + CastToTypeVectorProcessor.castToType(arg.buildVectorized(inputTypes), ExprType.STRING), + inputTypes.getMaxVectorSize() + ) + { + @Override + public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) + { + try { + longs[i] = Long.parseLong(strings[i], radix); + outputNulls[i] = false; + } + catch (NumberFormatException e) { + longs[i] = 0L; + outputNulls[i] = NullHandling.sqlCompatible(); + } + } + }; + + return (VectorExprProcessor) processor; + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java index 652a59033fab..0bd58aa4058b 100644 --- a/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/RowBasedStorageAdapterTest.java @@ -86,7 +86,7 @@ public class RowBasedStorageAdapterTest } ); - // Processors used by the "allProcessors" tasks. + // VectorProcessors used by the "allProcessors" tasks. private static final LinkedHashMap>> PROCESSORS = new LinkedHashMap<>(); @BeforeClass From 9a42ef1662663b060030079a9b945375f5b7d51f Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 18 Sep 2020 17:26:04 -0700 Subject: [PATCH 11/17] reduce boilerplate --- .../query/SqlExpressionBenchmark.java | 18 +- ...ubleOutLongInFunctionVectorProcessor.java} | 4 +- .../vector/VectorComparisonProcessors.java | 284 ++--- .../expr/vector/VectorMathProcessors.java | 966 ++++++++---------- 4 files changed, 562 insertions(+), 710 deletions(-) rename core/src/main/java/org/apache/druid/math/expr/vector/{LongOutDoubleInFunctionVectorProcessor.java => DoubleOutLongInFunctionVectorProcessor.java} (92%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index a4549367bdab..43d803938548 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -191,7 +191,7 @@ public String getFormatString() @Param({ // non-expression reference // "0", - "1", +// "1", // "2", // "3", // "4", @@ -199,12 +199,12 @@ public String getFormatString() // "6", // expressions "7", -// "8", -// "9", -// "10", -// "11", -// "12", -// "13", + "8", + "9", + "10", + "11", + "12", + "13", // "14", // "15", // "16", @@ -215,8 +215,8 @@ public String getFormatString() // "21", // "22", // "23", -// "24", -// "25" + "24", + "25" }) private String query; diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java similarity index 92% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java index d8045b5855dc..3f3179f3640f 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutDoubleInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java @@ -24,10 +24,10 @@ /** * specialized {@link UnivariateFunctionVectorProcessor} for processing (long[]) -> double[] */ -public abstract class LongOutDoubleInFunctionVectorProcessor +public abstract class DoubleOutLongInFunctionVectorProcessor extends UnivariateFunctionVectorProcessor { - public LongOutDoubleInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public DoubleOutLongInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new double[maxVectorSize]); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java index c82958880d86..f57088a8746a 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java @@ -21,8 +21,6 @@ import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; -import org.apache.druid.math.expr.ExprType; -import org.apache.druid.math.expr.Exprs; public class VectorComparisonProcessors { @@ -37,17 +35,14 @@ public static VectorExprProcessor equal( Expr right ) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return VectorMathProcessors.makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -55,12 +50,11 @@ public long apply(long left, long right) { return Evals.asLong(left == right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -68,14 +62,11 @@ public double apply(long left, double right) { return Evals.asDouble(left == right); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -83,12 +74,11 @@ public double apply(double left, long right) { return Evals.asDouble(left == right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -96,13 +86,8 @@ public double apply(double left, double right) { return Evals.asDouble(left == right); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor notEqual( @@ -111,17 +96,14 @@ public static VectorExprProcessor notEqual( Expr right ) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return VectorMathProcessors.makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -129,12 +111,11 @@ public long apply(long left, long right) { return Evals.asLong(left != right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -142,14 +123,11 @@ public double apply(long left, double right) { return Evals.asDouble(left != right); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -157,12 +135,11 @@ public double apply(double left, long right) { return Evals.asDouble(left != right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -170,13 +147,8 @@ public double apply(double left, double right) { return Evals.asDouble(left != right); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor greaterThanOrEqual( @@ -185,17 +157,14 @@ public static VectorExprProcessor greaterThanOrEqual( Expr right ) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return VectorMathProcessors.makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -203,12 +172,11 @@ public long apply(long left, long right) { return Evals.asLong(left >= right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -216,14 +184,11 @@ public double apply(long left, double right) { return Evals.asDouble(Double.compare(left, right) >= 0); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -231,12 +196,11 @@ public double apply(double left, long right) { return Evals.asDouble(Double.compare(left, right) >= 0); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -244,13 +208,8 @@ public double apply(double left, double right) { return Evals.asDouble(Double.compare(left, right) >= 0); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor greaterThan( @@ -259,17 +218,14 @@ public static VectorExprProcessor greaterThan( Expr right ) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return VectorMathProcessors.makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -277,12 +233,11 @@ public long apply(long left, long right) { return Evals.asLong(left > right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -290,14 +245,11 @@ public double apply(long left, double right) { return Evals.asDouble(Double.compare(left, right) > 0); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -305,12 +257,11 @@ public double apply(double left, long right) { return Evals.asDouble(Double.compare(left, right) > 0); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -318,13 +269,8 @@ public double apply(double left, double right) { return Evals.asDouble(Double.compare(left, right) > 0); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor lessThanOrEqual( @@ -333,17 +279,14 @@ public static VectorExprProcessor lessThanOrEqual( Expr right ) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return VectorMathProcessors.makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -351,12 +294,11 @@ public long apply(long left, long right) { return Evals.asLong(left <= right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -364,14 +306,11 @@ public double apply(long left, double right) { return Evals.asDouble(Double.compare(left, right) <= 0); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -379,12 +318,11 @@ public double apply(double left, long right) { return Evals.asDouble(Double.compare(left, right) <= 0); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -392,13 +330,8 @@ public double apply(double left, double right) { return Evals.asDouble(Double.compare(left, right) <= 0); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor lessThan( @@ -407,17 +340,14 @@ public static VectorExprProcessor lessThan( Expr right ) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return VectorMathProcessors.makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -425,12 +355,11 @@ public long apply(long left, long right) { return Evals.asLong(left < right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -438,14 +367,11 @@ public double apply(long left, double right) { return Evals.asDouble(Double.compare(left, right) < 0); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -453,12 +379,11 @@ public double apply(double left, long right) { return Evals.asDouble(Double.compare(left, right) < 0); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -466,12 +391,7 @@ public double apply(double left, double right) { return Evals.asDouble(Double.compare(left, right) < 0); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java index 95c73f17dc6c..437b6d25a136 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java @@ -25,21 +25,111 @@ import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.Exprs; +import java.util.function.Supplier; + public class VectorMathProcessors { - public static VectorExprProcessor plus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + /** + * Make a 1 argument math processor with the following type rules + * long -> long + * double -> double + */ + public static VectorExprProcessor makeMathProcessor( + Expr.VectorInputBindingTypes inputTypes, + Expr arg, + Supplier longOutLongInSupplier, + Supplier doubleOutDoubleInSupplier + ) + { + final ExprType inputType = arg.getOutputType(inputTypes); + + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = longOutLongInSupplier.get(); + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = doubleOutDoubleInSupplier.get(); + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + /** + * Make a 1 argument math processor with the following type rules + * long -> double + * double -> double + */ + public static VectorExprProcessor makeDoubleMathProcessor( + Expr.VectorInputBindingTypes inputTypes, + Expr arg, + Supplier doubleOutLongInSupplier, + Supplier doubleOutDoubleInSupplier + ) + { + final ExprType inputType = arg.getOutputType(inputTypes); + + VectorExprProcessor processor = null; + if (ExprType.LONG.equals(inputType)) { + processor = doubleOutLongInSupplier.get(); + } else if (ExprType.DOUBLE.equals(inputType)) { + processor = doubleOutDoubleInSupplier.get(); + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + /** + * Make a 2 argument, math processor with the following type rules + * long, long -> long + * long, double -> double + * double, long -> double + * double, double -> double + */ + public static VectorExprProcessor makeMathProcessor( + Expr.VectorInputBindingTypes inputTypes, + Expr left, + Expr right, + Supplier longOutLongsInProcessor, + Supplier doubleOutLongDoubleInProcessor, + Supplier doubleOutDoubleLongInProcessor, + Supplier doubleOutDoublesInProcessor + ) { final ExprType leftType = left.getOutputType(inputTypes); final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; + VectorExprProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + processor = longOutLongsInProcessor.get(); + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = doubleOutLongDoubleInProcessor.get(); + } + } else if (ExprType.DOUBLE.equals(leftType)) { + if (ExprType.LONG.equals(rightType)) { + processor = doubleOutDoubleLongInProcessor.get(); + } else if (ExprType.DOUBLE.equals(rightType)) { + processor = doubleOutDoublesInProcessor.get(); + } + } + if (processor == null) { + throw Exprs.cannotVectorize(); + } + return (VectorExprProcessor) processor; + } + + public static VectorExprProcessor plus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + { + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -47,12 +137,11 @@ public long apply(long left, long right) { return left + right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -60,14 +149,11 @@ public double apply(long left, double right) { return (double) left + right; } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -75,12 +161,11 @@ public double apply(double left, long right) { return left + (double) right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -88,28 +173,20 @@ public double apply(double left, double right) { return left + right; } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor minus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -117,12 +194,11 @@ public long apply(long left, long right) { return left - right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -130,14 +206,11 @@ public double apply(long left, double right) { return (double) left - right; } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -145,12 +218,11 @@ public double apply(double left, long right) { return left - (double) right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -158,28 +230,20 @@ public double apply(double left, double right) { return left - right; } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor multiply(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -187,12 +251,11 @@ public long apply(long left, long right) { return left * right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -200,14 +263,11 @@ public double apply(long left, double right) { return (double) left * right; } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -215,12 +275,11 @@ public double apply(double left, long right) { return left * (double) right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -228,28 +287,20 @@ public double apply(double left, double right) { return left * right; } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor divide(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -257,12 +308,11 @@ public long apply(long left, long right) { return left / right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -270,14 +320,11 @@ public double apply(long left, double right) { return (double) left / right; } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -285,12 +332,11 @@ public double apply(double left, long right) { return left / (double) right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -298,28 +344,20 @@ public double apply(double left, double right) { return left / right; } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor modulo(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -327,12 +365,11 @@ public long apply(long left, long right) { return left % right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -340,14 +377,11 @@ public double apply(long left, double right) { return (double) left % right; } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -355,12 +389,11 @@ public double apply(double left, long right) { return left % (double) right; } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -368,59 +401,50 @@ public double apply(double left, double right) { return left % right; } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor negate(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - final ExprType inputType = arg.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutLongInFunctionVectorProcessor(arg.buildVectorized(inputTypes), maxVectorSize) - { - @Override - public long apply(long input) + return makeMathProcessor( + inputTypes, + arg, + () -> new LongOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) { - return -input; - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor(arg.buildVectorized(inputTypes), maxVectorSize) - { - @Override - public double apply(double input) + @Override + public long apply(long input) + { + return -input; + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) { - return -input; + @Override + public double apply(double input) + { + return -input; + } } - }; - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor power(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -428,12 +452,11 @@ public long apply(long left, long right) { return LongMath.pow(left, Ints.checkedCast(right)); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -441,14 +464,11 @@ public double apply(long left, double right) { return Math.pow(left, right); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -456,12 +476,11 @@ public double apply(double left, long right) { return Math.pow(left, right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -469,13 +488,8 @@ public double apply(double left, double right) { return Math.pow(left, right); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor doublePower(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) @@ -506,17 +520,14 @@ public double apply(long left, long right) public static VectorExprProcessor max(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -524,12 +535,11 @@ public long apply(long left, long right) { return Math.max(left, right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -537,14 +547,11 @@ public double apply(long left, double right) { return Math.max(left, right); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -552,12 +559,11 @@ public double apply(double left, long right) { return Math.max(left, right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -565,28 +571,20 @@ public double apply(double left, double right) { return Math.max(left, right); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor min(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { - final ExprType leftType = left.getOutputType(inputTypes); - final ExprType rightType = right.getOutputType(inputTypes); - - final int maxVectorSize = inputTypes.getMaxVectorSize(); - BivariateFunctionVectorProcessor processor = null; - if (ExprType.LONG.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new LongOutLongsInFunctionVectorProcessor( + return makeMathProcessor( + inputTypes, + left, + right, + () -> new LongOutLongsInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -594,12 +592,11 @@ public long apply(long left, long right) { return Math.min(left, right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutLongDoubleInFunctionVectorProcessor( + }, + () -> new DoubleOutLongDoubleInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -607,14 +604,11 @@ public double apply(long left, double right) { return Math.min(left, right); } - }; - } - } else if (ExprType.DOUBLE.equals(leftType)) { - if (ExprType.LONG.equals(rightType)) { - processor = new DoubleOutDoubleLongInFunctionVectorProcessor( + }, + () -> new DoubleOutDoubleLongInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -622,12 +616,11 @@ public double apply(double left, long right) { return Math.min(left, right); } - }; - } else if (ExprType.DOUBLE.equals(rightType)) { - processor = new DoubleOutDoublesInFunctionVectorProcessor( + }, + () -> new DoubleOutDoublesInFunctionVectorProcessor( left.buildVectorized(inputTypes), right.buildVectorized(inputTypes), - maxVectorSize + inputTypes.getMaxVectorSize() ) { @Override @@ -635,309 +628,248 @@ public double apply(double left, double right) { return Math.min(left, right); } - }; - } - } - if (processor == null) { - throw Exprs.cannotVectorize(); - } - return (VectorExprProcessor) processor; + } + ); } public static VectorExprProcessor atan(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.atan(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.atan(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.atan(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.atan(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor cos(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.cos(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.cos(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cos(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cos(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor cosh(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.cosh(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.cosh(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cosh(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cosh(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor cot(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.cos(input) / Math.sin(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.cos(input) / Math.sin(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.cos(input) / Math.sin(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.cos(input) / Math.sin(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor sin(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.sin(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.sin(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.sin(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.sin(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor sinh(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.sinh(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.sinh(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.sinh(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.sinh(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor tan(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.tan(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.tan(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.tan(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.tan(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } public static VectorExprProcessor tanh(Expr.VectorInputBindingTypes inputTypes, Expr arg) { - ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; - if (ExprType.LONG.equals(inputType)) { - processor = new LongOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(long input) - { - return Math.tanh(input); - } - }; - } else if (ExprType.DOUBLE.equals(inputType)) { - processor = new DoubleOutDoubleInFunctionVectorProcessor( - arg.buildVectorized(inputTypes), - inputTypes.getMaxVectorSize() - ) - { - @Override - public double apply(double input) - { - return Math.tanh(input); + return makeDoubleMathProcessor( + inputTypes, + arg, + () -> new DoubleOutLongInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(long input) + { + return Math.tanh(input); + } + }, + () -> new DoubleOutDoubleInFunctionVectorProcessor( + arg.buildVectorized(inputTypes), + inputTypes.getMaxVectorSize() + ) + { + @Override + public double apply(double input) + { + return Math.tanh(input); + } } - }; - } - - if (processor == null) { - throw Exprs.cannotVectorize(); - } - - return (VectorExprProcessor) processor; + ); } private VectorMathProcessors() From 2b097b601d586016f396ef8919a03e1cfddff240 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 18 Sep 2020 17:27:09 -0700 Subject: [PATCH 12/17] oops --- .../query/SqlExpressionBenchmark.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 43d803938548..0a098783688c 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -190,13 +190,13 @@ public String getFormatString() @Param({ // non-expression reference -// "0", -// "1", -// "2", -// "3", -// "4", -// "5", -// "6", + "0", + "1", + "2", + "3", + "4", + "5", + "6", // expressions "7", "8", @@ -205,16 +205,16 @@ public String getFormatString() "11", "12", "13", -// "14", -// "15", -// "16", -// "17", -// "18", -// "19", -// "20", -// "21", -// "22", -// "23", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", "24", "25" }) From e8d240a71c339f44b96beca5a749e8e237bb2f2d Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Fri, 18 Sep 2020 19:29:20 -0700 Subject: [PATCH 13/17] more naming adjustments --- .../query/SqlExpressionBenchmark.java | 3 +- .../apache/druid/math/expr/ApplyFunction.java | 4 +- .../math/expr/BinaryLogicalOperatorExpr.java | 14 ++--- .../math/expr/BinaryMathOperatorExpr.java | 14 ++--- .../apache/druid/math/expr/ConstantExpr.java | 12 ++-- .../java/org/apache/druid/math/expr/Expr.java | 15 ++++- .../org/apache/druid/math/expr/Function.java | 37 ++++++------ .../druid/math/expr/FunctionalExpr.java | 8 +-- .../druid/math/expr/IdentifierExpr.java | 26 ++++----- .../druid/math/expr/UnaryOperatorExpr.java | 4 +- .../BivariateFunctionVectorProcessor.java | 18 +++--- .../vector/CastToDoubleVectorProcessor.java | 8 +-- .../vector/CastToLongVectorProcessor.java | 8 +-- .../vector/CastToStringVectorProcessor.java | 8 +-- .../vector/CastToTypeVectorProcessor.java | 12 ++-- ...bleOutDoubleInFunctionVectorProcessor.java | 6 +- ...utDoubleLongInFunctionVectorProcessor.java | 8 +-- ...leOutDoublesInFunctionVectorProcessor.java | 8 +-- ...utLongDoubleInFunctionVectorProcessor.java | 8 +-- ...oubleOutLongInFunctionVectorProcessor.java | 6 +- ...ubleOutLongsInFunctionVectorProcessor.java | 8 +-- ...xprEval.java => ExprEvalDoubleVector.java} | 4 +- ...rExprEval.java => ExprEvalLongVector.java} | 4 +- ...xprEval.java => ExprEvalStringVector.java} | 4 +- ...ectorExprEval.java => ExprEvalVector.java} | 6 +- ...rocessor.java => ExprVectorProcessor.java} | 4 +- .../LongOutLongInFunctionVectorProcessor.java | 6 +- ...LongOutLongsInFunctionVectorProcessor.java | 8 +-- ...ongOutStringInFunctionVectorProcessor.java | 6 +- ...ivariateFunctionVectorObjectProcessor.java | 12 ++-- .../UnivariateFunctionVectorProcessor.java | 12 ++-- .../vector/VectorComparisonProcessors.java | 12 ++-- .../expr/vector/VectorMathProcessors.java | 56 +++++++++---------- .../math/expr/vector/VectorProcessors.java | 36 ++++++------ .../druid/math/expr/VectorExprSanityTest.java | 4 +- .../expression/TimestampFloorExprMacro.java | 8 +-- .../ExpressionVectorObjectSelector.java | 6 +- .../virtual/ExpressionVectorSelectors.java | 6 +- .../ExpressionVectorValueSelector.java | 6 +- .../virtual/ExpressionVirtualColumn.java | 12 ++++ 40 files changed, 235 insertions(+), 212 deletions(-) rename core/src/main/java/org/apache/druid/math/expr/vector/{DoubleVectorExprEval.java => ExprEvalDoubleVector.java} (93%) rename core/src/main/java/org/apache/druid/math/expr/vector/{LongVectorExprEval.java => ExprEvalLongVector.java} (93%) rename core/src/main/java/org/apache/druid/math/expr/vector/{StringVectorExprEval.java => ExprEvalStringVector.java} (95%) rename core/src/main/java/org/apache/druid/math/expr/vector/{VectorExprEval.java => ExprEvalVector.java} (92%) rename core/src/main/java/org/apache/druid/math/expr/vector/{VectorExprProcessor.java => ExprVectorProcessor.java} (91%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 0a098783688c..8ccfd73a8e39 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -354,8 +354,9 @@ public void checkSanity() throws Exception Assert.assertTrue(vectorizedYielder.isDone()); Assert.assertTrue(nonVectorizedYielder.isDone()); } - catch (Throwable ignored) { + catch (Throwable t) { // the show must go on + log.warn(t.getMessage()); } } } diff --git a/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java b/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java index 275da27d07ec..73fe42153172 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java +++ b/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java @@ -28,7 +28,7 @@ import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.UOE; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import javax.annotation.Nullable; import java.util.ArrayList; @@ -56,7 +56,7 @@ default boolean canVectorize(Expr.InputBindingTypes inputTypes, Expr lambda, Lis return false; } - default VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, Expr lambda, List args) + default ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, Expr lambda, List args) { throw new UOE("%s is not vectorized", name()); } diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java index b207ed1fe972..e0a34892ee5b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java @@ -20,8 +20,8 @@ package org.apache.druid.math.expr; import org.apache.druid.java.util.common.guava.Comparators; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.VectorComparisonProcessors; -import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nullable; import java.util.Objects; @@ -77,7 +77,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorComparisonProcessors.lessThan(inputTypes, left, right); } @@ -133,7 +133,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorComparisonProcessors.lessThanOrEqual(inputTypes, left, right); } @@ -188,7 +188,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorComparisonProcessors.greaterThan(inputTypes, left, right); } @@ -244,7 +244,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorComparisonProcessors.greaterThanOrEqual(inputTypes, left, right); } @@ -299,7 +299,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorComparisonProcessors.equal(inputTypes, left, right); } @@ -354,7 +354,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorComparisonProcessors.notEqual(inputTypes, left, right); } diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java index a0d496640e0d..39b5e1360c36 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryMathOperatorExpr.java @@ -22,7 +22,7 @@ import com.google.common.math.LongMath; import com.google.common.primitives.Ints; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.VectorMathProcessors; import javax.annotation.Nullable; @@ -67,7 +67,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.plus(inputTypes, left, right); } @@ -105,7 +105,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.minus(inputTypes, left, right); } @@ -143,7 +143,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.multiply(inputTypes, left, right); } @@ -181,7 +181,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.divide(inputTypes, left, right); } @@ -219,7 +219,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.power(inputTypes, left, right); } @@ -257,7 +257,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.modulo(inputTypes, left, right); } diff --git a/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java b/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java index c8c7f2b3909a..d9075f8c62eb 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/ConstantExpr.java @@ -23,7 +23,7 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.VectorProcessors; import javax.annotation.Nullable; @@ -142,7 +142,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorProcessors.constantLong(value, inputTypes.getMaxVectorSize()); } @@ -187,7 +187,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorProcessors.constantLong(null, inputTypes.getMaxVectorSize()); } @@ -297,7 +297,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorProcessors.constantDouble(value, inputTypes.getMaxVectorSize()); } @@ -341,7 +341,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorProcessors.constantDouble(null, inputTypes.getMaxVectorSize()); } @@ -453,7 +453,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorProcessors.constantString(value, inputTypes.getMaxVectorSize()); } diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index f5cd8e4d43b5..c7834c2b4d65 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -24,7 +24,7 @@ import com.google.common.collect.Sets; import org.apache.druid.annotations.SubclassesMustOverrideEqualsAndHashCode; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import javax.annotation.Nullable; import java.util.ArrayList; @@ -142,12 +142,21 @@ default ExprType getOutputType(InputBindingTypes inputTypes) return null; } + /** + * Check if an expression can be 'vectorized', for a given set of inputs. If this method returns true, + * {@link #buildVectorized} is expected to produce a {@link ExprVectorProcessor} which can evaluate values in batches + * to use with vectorized query engines. + */ default boolean canVectorize(InputBindingTypes inputTypes) { return false; } - default VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + /** + * Builds a 'vectorized' expression processor, that can operate on batches of input values for use in vectorized + * query engines. + */ + default ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { throw Exprs.cannotVectorize(this); } @@ -230,7 +239,7 @@ interface ObjectBinding } /** - * Mechanism to supply batches of input values to a {@link VectorExprProcessor} for optimized processing. Mirrors + * Mechanism to supply batches of input values to a {@link ExprVectorProcessor} for optimized processing. Mirrors * the vectorized column selector interfaces, and includes {@link ExprType} information about all input bindings * which exist */ diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index c238dc6e224f..afdd812e06b6 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -27,7 +27,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.UOE; import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.VectorMathProcessors; import org.apache.druid.math.expr.vector.VectorProcessors; import org.joda.time.DateTime; @@ -122,7 +122,7 @@ default boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) return false; } - default VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + default ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { throw new UOE("%s is not vectorized", name()); } @@ -536,11 +536,12 @@ public ExprEval apply(List args, Expr.ObjectBinding bindings) @Override public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) { - return (args.size() == 1 || args.get(1).isLiteral()) && inputTypes.canVectorize(args); + return (args.size() == 1 || (args.get(1).isLiteral() && args.get(1).getLiteralValue() instanceof Number)) && + inputTypes.canVectorize(args); } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { if (args.size() == 1 || args.get(1).isLiteral()) { final int radix = args.size() == 1 ? 10 : ((Number) args.get(1).getLiteralValue()).intValue(); @@ -589,7 +590,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorProcessors.constantDouble(PI, inputTypes.getMaxVectorSize()); } @@ -667,7 +668,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.atan(inputTypes, args.get(0)); } @@ -724,7 +725,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.cos(inputTypes, args.get(0)); } @@ -751,7 +752,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.cosh(inputTypes, args.get(0)); } @@ -778,7 +779,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.cot(inputTypes, args.get(0)); } @@ -811,7 +812,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.divide(inputTypes, args.get(0), args.get(1)); } @@ -1076,7 +1077,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.sin(inputTypes, args.get(0)); } @@ -1103,7 +1104,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.sinh(inputTypes, args.get(0)); } @@ -1145,7 +1146,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.tan(inputTypes, args.get(0)); } @@ -1172,7 +1173,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.tanh(inputTypes, args.get(0)); } @@ -1310,7 +1311,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.max(inputTypes, args.get(0), args.get(1)); } @@ -1343,7 +1344,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.min(inputTypes, args.get(0), args.get(1)); } @@ -1385,7 +1386,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return VectorMathProcessors.doublePower(inputTypes, args.get(0), args.get(1)); } @@ -1491,7 +1492,7 @@ public boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) } @Override - public VectorExprProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) + public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { return CastToTypeVectorProcessor.castToType( args.get(0).buildVectorized(inputTypes), diff --git a/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java b/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java index 5323c87d1928..80bc1e7ab2e6 100644 --- a/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/FunctionalExpr.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import javax.annotation.Nullable; import java.util.List; @@ -84,7 +84,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return expr.buildVectorized(inputTypes); } @@ -190,7 +190,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return function.asVectorProcessor(inputTypes, args); } @@ -322,7 +322,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return function.asVectorProcessor(inputTypes, lambdaExpr, argsExpr); } diff --git a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java index d2a8cc2ae382..fd8d7aed1756 100644 --- a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java @@ -21,11 +21,11 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.DoubleVectorExprEval; -import org.apache.druid.math.expr.vector.LongVectorExprEval; -import org.apache.druid.math.expr.vector.StringVectorExprEval; -import org.apache.druid.math.expr.vector.VectorExprEval; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprEvalDoubleVector; +import org.apache.druid.math.expr.vector.ExprEvalLongVector; +import org.apache.druid.math.expr.vector.ExprEvalStringVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import javax.annotation.Nullable; import java.util.Objects; @@ -150,7 +150,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { ExprType inputType = inputTypes.getType(binding); @@ -162,27 +162,27 @@ public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes return new IdentifierVectorProcessor(ExprType.LONG) { @Override - public VectorExprEval evalVector(VectorInputBinding bindings) + public ExprEvalVector evalVector(VectorInputBinding bindings) { - return new LongVectorExprEval(bindings.getLongVector(binding), bindings.getNullVector(binding)); + return new ExprEvalLongVector(bindings.getLongVector(binding), bindings.getNullVector(binding)); } }; case DOUBLE: return new IdentifierVectorProcessor(ExprType.DOUBLE) { @Override - public VectorExprEval evalVector(VectorInputBinding bindings) + public ExprEvalVector evalVector(VectorInputBinding bindings) { - return new DoubleVectorExprEval(bindings.getDoubleVector(binding), bindings.getNullVector(binding)); + return new ExprEvalDoubleVector(bindings.getDoubleVector(binding), bindings.getNullVector(binding)); } }; case STRING: return new IdentifierVectorProcessor(ExprType.STRING) { @Override - public VectorExprEval evalVector(VectorInputBinding bindings) + public ExprEvalVector evalVector(VectorInputBinding bindings) { - return new StringVectorExprEval(bindings.getObjectVector(binding)); + return new ExprEvalStringVector(bindings.getObjectVector(binding)); } }; default: @@ -210,7 +210,7 @@ public int hashCode() } } -abstract class IdentifierVectorProcessor implements VectorExprProcessor +abstract class IdentifierVectorProcessor implements ExprVectorProcessor { private final ExprType outputType; diff --git a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java index 05e1b8a4bbff..c741a5a808ec 100644 --- a/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/UnaryOperatorExpr.java @@ -23,7 +23,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.VectorMathProcessors; import javax.annotation.Nullable; @@ -145,7 +145,7 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { return VectorMathProcessors.negate(inputTypes, expr); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java index 41f8376b8e52..6ff6355e0a57 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/BivariateFunctionVectorProcessor.java @@ -27,17 +27,17 @@ * checking the vector themselves for nulls) */ public abstract class BivariateFunctionVectorProcessor - implements VectorExprProcessor + implements ExprVectorProcessor { - final VectorExprProcessor left; - final VectorExprProcessor right; + final ExprVectorProcessor left; + final ExprVectorProcessor right; final int maxVectorSize; final boolean[] outNulls; final TOutput outValues; protected BivariateFunctionVectorProcessor( - VectorExprProcessor left, - VectorExprProcessor right, + ExprVectorProcessor left, + ExprVectorProcessor right, int maxVectorSize, TOutput outValues ) @@ -50,10 +50,10 @@ protected BivariateFunctionVectorProcessor( } @Override - public final VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public final ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final VectorExprEval lhs = left.evalVector(bindings); - final VectorExprEval rhs = right.evalVector(bindings); + final ExprEvalVector lhs = left.evalVector(bindings); + final ExprEvalVector rhs = right.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); final boolean[] leftNulls = lhs.getNullVector(); @@ -83,5 +83,5 @@ public final VectorExprEval evalVector(Expr.VectorInputBinding bindings abstract void processIndex(TLeftInput leftInput, TRightInput rightInput, int i); - abstract VectorExprEval asEval(); + abstract ExprEvalVector asEval(); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java index 2afc6bdfe611..2cb82c7c1a72 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToDoubleVectorProcessor.java @@ -24,16 +24,16 @@ public final class CastToDoubleVectorProcessor extends CastToTypeVectorProcessor { - public CastToDoubleVectorProcessor(VectorExprProcessor delegate) + public CastToDoubleVectorProcessor(ExprVectorProcessor delegate) { super(delegate); } @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - VectorExprEval result = delegate.evalVector(bindings); - return new DoubleVectorExprEval(result.getDoubleVector(), result.getNullVector()); + ExprEvalVector result = delegate.evalVector(bindings); + return new ExprEvalDoubleVector(result.getDoubleVector(), result.getNullVector()); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java index 614e14a9210a..65d5812f5206 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToLongVectorProcessor.java @@ -24,16 +24,16 @@ public final class CastToLongVectorProcessor extends CastToTypeVectorProcessor { - public CastToLongVectorProcessor(VectorExprProcessor delegate) + public CastToLongVectorProcessor(ExprVectorProcessor delegate) { super(delegate); } @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - VectorExprEval result = delegate.evalVector(bindings); - return new LongVectorExprEval(result.getLongVector(), result.getNullVector()); + ExprEvalVector result = delegate.evalVector(bindings); + return new ExprEvalLongVector(result.getLongVector(), result.getNullVector()); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java index e5e09098810d..5903d17ac25a 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java @@ -24,16 +24,16 @@ public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor { - public CastToStringVectorProcessor(VectorExprProcessor delegate) + public CastToStringVectorProcessor(ExprVectorProcessor delegate) { super(delegate); } @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - VectorExprEval result = delegate.evalVector(bindings); - return new StringVectorExprEval(result.asObjectVector(ExprType.STRING)); + ExprEvalVector result = delegate.evalVector(bindings); + return new ExprEvalStringVector(result.asObjectVector(ExprType.STRING)); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java index a9a04cdc9a82..b15b370a2f42 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToTypeVectorProcessor.java @@ -22,18 +22,18 @@ import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.Exprs; -public abstract class CastToTypeVectorProcessor implements VectorExprProcessor +public abstract class CastToTypeVectorProcessor implements ExprVectorProcessor { - protected final VectorExprProcessor delegate; + protected final ExprVectorProcessor delegate; - protected CastToTypeVectorProcessor(VectorExprProcessor delegate) + protected CastToTypeVectorProcessor(ExprVectorProcessor delegate) { this.delegate = delegate; } - public static VectorExprProcessor castToType(VectorExprProcessor delegate, ExprType type) + public static ExprVectorProcessor castToType(ExprVectorProcessor delegate, ExprType type) { - final VectorExprProcessor caster; + final ExprVectorProcessor caster; if (delegate.getOutputType() == type) { caster = delegate; } else { @@ -51,6 +51,6 @@ public static VectorExprProcessor castToType(VectorExprProcessor deleg throw Exprs.cannotVectorize(); } } - return (VectorExprProcessor) caster; + return (ExprVectorProcessor) caster; } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java index 1fb4edab3e99..e78f5bd99396 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleInFunctionVectorProcessor.java @@ -27,7 +27,7 @@ public abstract class DoubleOutDoubleInFunctionVectorProcessor extends UnivariateFunctionVectorProcessor { - public DoubleOutDoubleInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public DoubleOutDoubleInFunctionVectorProcessor(ExprVectorProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new double[maxVectorSize]); } @@ -47,8 +47,8 @@ final void processIndex(double[] input, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new DoubleVectorExprEval(outValues, outNulls); + return new ExprEvalDoubleVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java index 6a14607d3a88..9cd0cc3fdf34 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoubleLongInFunctionVectorProcessor.java @@ -28,8 +28,8 @@ public abstract class DoubleOutDoubleLongInFunctionVectorProcessor extends BivariateFunctionVectorProcessor { public DoubleOutDoubleLongInFunctionVectorProcessor( - VectorExprProcessor left, - VectorExprProcessor right, + ExprVectorProcessor left, + ExprVectorProcessor right, int maxVectorSize ) { @@ -51,8 +51,8 @@ final void processIndex(double[] leftInput, long[] rightInput, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new DoubleVectorExprEval(outValues, outNulls); + return new ExprEvalDoubleVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java index 6a9f194cb2b5..fb98716bdd17 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutDoublesInFunctionVectorProcessor.java @@ -28,8 +28,8 @@ public abstract class DoubleOutDoublesInFunctionVectorProcessor extends BivariateFunctionVectorProcessor { public DoubleOutDoublesInFunctionVectorProcessor( - VectorExprProcessor left, - VectorExprProcessor right, + ExprVectorProcessor left, + ExprVectorProcessor right, int maxVectorSize ) { @@ -51,8 +51,8 @@ final void processIndex(double[] leftInput, double[] rightInput, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new DoubleVectorExprEval(outValues, outNulls); + return new ExprEvalDoubleVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java index a66e8bbfc38c..32ef0b3d197c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongDoubleInFunctionVectorProcessor.java @@ -28,8 +28,8 @@ public abstract class DoubleOutLongDoubleInFunctionVectorProcessor extends BivariateFunctionVectorProcessor { public DoubleOutLongDoubleInFunctionVectorProcessor( - VectorExprProcessor left, - VectorExprProcessor right, + ExprVectorProcessor left, + ExprVectorProcessor right, int maxVectorSize ) { @@ -51,8 +51,8 @@ final void processIndex(long[] leftInput, double[] rightInput, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new DoubleVectorExprEval(outValues, outNulls); + return new ExprEvalDoubleVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java index 3f3179f3640f..b85995d38438 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongInFunctionVectorProcessor.java @@ -27,7 +27,7 @@ public abstract class DoubleOutLongInFunctionVectorProcessor extends UnivariateFunctionVectorProcessor { - public DoubleOutLongInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public DoubleOutLongInFunctionVectorProcessor(ExprVectorProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new double[maxVectorSize]); } @@ -47,8 +47,8 @@ final void processIndex(long[] input, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new DoubleVectorExprEval(outValues, outNulls); + return new ExprEvalDoubleVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java index 26425f1b5d35..2af72bc86f0e 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/DoubleOutLongsInFunctionVectorProcessor.java @@ -28,8 +28,8 @@ public abstract class DoubleOutLongsInFunctionVectorProcessor extends BivariateFunctionVectorProcessor { public DoubleOutLongsInFunctionVectorProcessor( - VectorExprProcessor left, - VectorExprProcessor right, + ExprVectorProcessor left, + ExprVectorProcessor right, int maxVectorSize ) { @@ -51,8 +51,8 @@ final void processIndex(long[] leftInput, long[] rightInput, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new DoubleVectorExprEval(outValues, outNulls); + return new ExprEvalDoubleVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalDoubleVector.java similarity index 93% rename from core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalDoubleVector.java index dd2ef0a39da2..3eae05d4df36 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/DoubleVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalDoubleVector.java @@ -24,9 +24,9 @@ import java.util.Arrays; -public final class DoubleVectorExprEval extends VectorExprEval +public final class ExprEvalDoubleVector extends ExprEvalVector { - public DoubleVectorExprEval(double[] values, boolean[] nulls) + public ExprEvalDoubleVector(double[] values, boolean[] nulls) { super(values, nulls); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalLongVector.java similarity index 93% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalLongVector.java index ec0648f3cb6c..3f91da475445 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalLongVector.java @@ -25,9 +25,9 @@ import javax.annotation.Nullable; import java.util.Arrays; -public final class LongVectorExprEval extends VectorExprEval +public final class ExprEvalLongVector extends ExprEvalVector { - public LongVectorExprEval(long[] values, @Nullable boolean[] nulls) + public ExprEvalLongVector(long[] values, @Nullable boolean[] nulls) { super(values, nulls); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java similarity index 95% rename from core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java index c8536571c5e4..73a4a9501daf 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/StringVectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java @@ -26,7 +26,7 @@ import javax.annotation.Nullable; -public final class StringVectorExprEval extends VectorExprEval +public final class ExprEvalStringVector extends ExprEvalVector { @Nullable private long[] longs; @@ -36,7 +36,7 @@ public final class StringVectorExprEval extends VectorExprEval @Nullable private boolean[] numericNulls; - public StringVectorExprEval(String[] values) + public ExprEvalStringVector(String[] values) { super(values, null); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java similarity index 92% rename from core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java index 63bf50cfa987..7da17783e090 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprEval.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java @@ -26,18 +26,18 @@ import java.lang.reflect.Array; /** - * Result of {@link VectorExprProcessor#evalVector} which wraps the actual evaluated results of the operation over the + * Result of {@link ExprVectorProcessor#evalVector} which wraps the actual evaluated results of the operation over the * input vector(s). Methods to get actual results mirror vectorized value and object selectors. * * The generic parameter T should be the native java array type of the vector result (long[], String[], etc.) */ -public abstract class VectorExprEval +public abstract class ExprEvalVector { final T values; @Nullable final boolean[] nulls; - public VectorExprEval(T values, @Nullable boolean[] nulls) + public ExprEvalVector(T values, @Nullable boolean[] nulls) { this.values = values; this.nulls = nulls; diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprVectorProcessor.java similarity index 91% rename from core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ExprVectorProcessor.java index 88c2d672f2f3..24688124df2d 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorExprProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprVectorProcessor.java @@ -26,9 +26,9 @@ * Interface describing vectorized expression processors, which can be specialized using input type information to * produce optimized expression evaluators, which can operate on batches of primitive data with minimal object overhead */ -public interface VectorExprProcessor +public interface ExprVectorProcessor { - VectorExprEval evalVector(Expr.VectorInputBinding bindings); + ExprEvalVector evalVector(Expr.VectorInputBinding bindings); ExprType getOutputType(); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java index e4ad864a87fb..92a136c3b245 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongInFunctionVectorProcessor.java @@ -26,7 +26,7 @@ */ public abstract class LongOutLongInFunctionVectorProcessor extends UnivariateFunctionVectorProcessor { - public LongOutLongInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public LongOutLongInFunctionVectorProcessor(ExprVectorProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new long[maxVectorSize]); } @@ -46,8 +46,8 @@ final void processIndex(long[] input, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new LongVectorExprEval(outValues, outNulls); + return new ExprEvalLongVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java index 788668a487a8..7d85e881f2d4 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutLongsInFunctionVectorProcessor.java @@ -28,8 +28,8 @@ public abstract class LongOutLongsInFunctionVectorProcessor extends BivariateFunctionVectorProcessor { public LongOutLongsInFunctionVectorProcessor( - VectorExprProcessor left, - VectorExprProcessor right, + ExprVectorProcessor left, + ExprVectorProcessor right, int maxVectorSize ) { @@ -51,8 +51,8 @@ final void processIndex(long[] leftInput, long[] rightInput, int i) } @Override - final VectorExprEval asEval() + final ExprEvalVector asEval() { - return new LongVectorExprEval(outValues, outNulls); + return new ExprEvalLongVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java index 0537da260d07..22de162bf2e6 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java @@ -27,7 +27,7 @@ public abstract class LongOutStringInFunctionVectorProcessor extends UnivariateFunctionVectorObjectProcessor { - public LongOutStringInFunctionVectorProcessor(VectorExprProcessor processor, int maxVectorSize) + public LongOutStringInFunctionVectorProcessor(ExprVectorProcessor processor, int maxVectorSize) { super(processor, maxVectorSize, new long[maxVectorSize]); } @@ -39,8 +39,8 @@ public ExprType getOutputType() } @Override - public final VectorExprEval asEval() + public final ExprEvalVector asEval() { - return new LongVectorExprEval(outValues, outNulls); + return new ExprEvalLongVector(outValues, outNulls); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java index 6679c177efd0..02fa6dd3768c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorObjectProcessor.java @@ -25,15 +25,15 @@ * common machinery for processing single input operators and functions, which are backed by an object value instead of * a primitive value (so do not need to use the null vector, and instead can check the value vector itself for nulls) */ -public abstract class UnivariateFunctionVectorObjectProcessor implements VectorExprProcessor +public abstract class UnivariateFunctionVectorObjectProcessor implements ExprVectorProcessor { - final VectorExprProcessor processor; + final ExprVectorProcessor processor; final int maxVectorSize; final boolean[] outNulls; final TOutput outValues; public UnivariateFunctionVectorObjectProcessor( - VectorExprProcessor processor, + ExprVectorProcessor processor, int maxVectorSize, TOutput outValues ) @@ -45,9 +45,9 @@ public UnivariateFunctionVectorObjectProcessor( } @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final VectorExprEval lhs = processor.evalVector(bindings); + final ExprEvalVector lhs = processor.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); @@ -61,5 +61,5 @@ public VectorExprEval evalVector(Expr.VectorInputBinding bindings) public abstract void processIndex(TInput input, TOutput output, boolean[] outputNulls, int i); - public abstract VectorExprEval asEval(); + public abstract ExprEvalVector asEval(); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java index 10cde10b4c17..4db9c0eed828 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/UnivariateFunctionVectorProcessor.java @@ -26,15 +26,15 @@ * output, and are backed by a primitive value instead of an object value (and need to use the null vector instead of * checking the vector itself for nulls) */ -public abstract class UnivariateFunctionVectorProcessor implements VectorExprProcessor +public abstract class UnivariateFunctionVectorProcessor implements ExprVectorProcessor { - final VectorExprProcessor processor; + final ExprVectorProcessor processor; final int maxVectorSize; final boolean[] outNulls; final TOutput outValues; public UnivariateFunctionVectorProcessor( - VectorExprProcessor processor, + ExprVectorProcessor processor, int maxVectorSize, TOutput outValues ) @@ -46,9 +46,9 @@ public UnivariateFunctionVectorProcessor( } @Override - public final VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public final ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final VectorExprEval lhs = processor.evalVector(bindings); + final ExprEvalVector lhs = processor.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); final boolean[] inputNulls = lhs.getNullVector(); @@ -74,5 +74,5 @@ public final VectorExprEval evalVector(Expr.VectorInputBinding bindings abstract void processIndex(TInput input, int i); - abstract VectorExprEval asEval(); + abstract ExprEvalVector asEval(); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java index f57088a8746a..49abf5bbf5a2 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java @@ -29,7 +29,7 @@ private VectorComparisonProcessors() // No instantiation } - public static VectorExprProcessor equal( + public static ExprVectorProcessor equal( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right @@ -90,7 +90,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor notEqual( + public static ExprVectorProcessor notEqual( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right @@ -151,7 +151,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor greaterThanOrEqual( + public static ExprVectorProcessor greaterThanOrEqual( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right @@ -212,7 +212,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor greaterThan( + public static ExprVectorProcessor greaterThan( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right @@ -273,7 +273,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor lessThanOrEqual( + public static ExprVectorProcessor lessThanOrEqual( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right @@ -334,7 +334,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor lessThan( + public static ExprVectorProcessor lessThan( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java index 437b6d25a136..ea6db8f7a634 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorMathProcessors.java @@ -34,7 +34,7 @@ public class VectorMathProcessors * long -> long * double -> double */ - public static VectorExprProcessor makeMathProcessor( + public static ExprVectorProcessor makeMathProcessor( Expr.VectorInputBindingTypes inputTypes, Expr arg, Supplier longOutLongInSupplier, @@ -43,7 +43,7 @@ public static VectorExprProcessor makeMathProcessor( { final ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; + ExprVectorProcessor processor = null; if (ExprType.LONG.equals(inputType)) { processor = longOutLongInSupplier.get(); } else if (ExprType.DOUBLE.equals(inputType)) { @@ -52,7 +52,7 @@ public static VectorExprProcessor makeMathProcessor( if (processor == null) { throw Exprs.cannotVectorize(); } - return (VectorExprProcessor) processor; + return (ExprVectorProcessor) processor; } /** @@ -60,7 +60,7 @@ public static VectorExprProcessor makeMathProcessor( * long -> double * double -> double */ - public static VectorExprProcessor makeDoubleMathProcessor( + public static ExprVectorProcessor makeDoubleMathProcessor( Expr.VectorInputBindingTypes inputTypes, Expr arg, Supplier doubleOutLongInSupplier, @@ -69,7 +69,7 @@ public static VectorExprProcessor makeDoubleMathProcessor( { final ExprType inputType = arg.getOutputType(inputTypes); - VectorExprProcessor processor = null; + ExprVectorProcessor processor = null; if (ExprType.LONG.equals(inputType)) { processor = doubleOutLongInSupplier.get(); } else if (ExprType.DOUBLE.equals(inputType)) { @@ -78,7 +78,7 @@ public static VectorExprProcessor makeDoubleMathProcessor( if (processor == null) { throw Exprs.cannotVectorize(); } - return (VectorExprProcessor) processor; + return (ExprVectorProcessor) processor; } /** @@ -88,7 +88,7 @@ public static VectorExprProcessor makeDoubleMathProcessor( * double, long -> double * double, double -> double */ - public static VectorExprProcessor makeMathProcessor( + public static ExprVectorProcessor makeMathProcessor( Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right, @@ -100,7 +100,7 @@ public static VectorExprProcessor makeMathProcessor( { final ExprType leftType = left.getOutputType(inputTypes); final ExprType rightType = right.getOutputType(inputTypes); - VectorExprProcessor processor = null; + ExprVectorProcessor processor = null; if (ExprType.LONG.equals(leftType)) { if (ExprType.LONG.equals(rightType)) { processor = longOutLongsInProcessor.get(); @@ -117,10 +117,10 @@ public static VectorExprProcessor makeMathProcessor( if (processor == null) { throw Exprs.cannotVectorize(); } - return (VectorExprProcessor) processor; + return (ExprVectorProcessor) processor; } - public static VectorExprProcessor plus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor plus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -177,7 +177,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor minus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor minus(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -234,7 +234,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor multiply(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor multiply(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -291,7 +291,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor divide(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor divide(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -348,7 +348,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor modulo(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor modulo(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -405,7 +405,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor negate(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor negate(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeMathProcessor( inputTypes, @@ -435,7 +435,7 @@ public double apply(double input) ); } - public static VectorExprProcessor power(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor power(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -492,7 +492,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor doublePower(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor doublePower(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { BivariateFunctionVectorProcessor processor = null; if (ExprType.LONG.equals(left.getOutputType(inputTypes))) { @@ -513,12 +513,12 @@ public double apply(long left, long right) } if (processor != null) { - return (VectorExprProcessor) processor; + return (ExprVectorProcessor) processor; } return power(inputTypes, left, right); } - public static VectorExprProcessor max(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor max(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -575,7 +575,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor min(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) + public static ExprVectorProcessor min(Expr.VectorInputBindingTypes inputTypes, Expr left, Expr right) { return makeMathProcessor( inputTypes, @@ -632,7 +632,7 @@ public double apply(double left, double right) ); } - public static VectorExprProcessor atan(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor atan(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -662,7 +662,7 @@ public double apply(double input) ); } - public static VectorExprProcessor cos(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor cos(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -692,7 +692,7 @@ public double apply(double input) ); } - public static VectorExprProcessor cosh(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor cosh(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -722,7 +722,7 @@ public double apply(double input) ); } - public static VectorExprProcessor cot(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor cot(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -752,7 +752,7 @@ public double apply(double input) ); } - public static VectorExprProcessor sin(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor sin(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -782,7 +782,7 @@ public double apply(double input) ); } - public static VectorExprProcessor sinh(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor sinh(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -812,7 +812,7 @@ public double apply(double input) ); } - public static VectorExprProcessor tan(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor tan(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, @@ -842,7 +842,7 @@ public double apply(double input) ); } - public static VectorExprProcessor tanh(Expr.VectorInputBindingTypes inputTypes, Expr arg) + public static ExprVectorProcessor tanh(Expr.VectorInputBindingTypes inputTypes, Expr arg) { return makeDoubleMathProcessor( inputTypes, diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index d8c873272310..11aced2ce6ac 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -28,17 +28,17 @@ public class VectorProcessors { - public static VectorExprProcessor constantString(@Nullable String constant, int maxVectorSize) + public static ExprVectorProcessor constantString(@Nullable String constant, int maxVectorSize) { final String[] strings = new String[maxVectorSize]; Arrays.fill(strings, constant); - final StringVectorExprEval eval = new StringVectorExprEval(strings); - return new VectorExprProcessor() + final ExprEvalStringVector eval = new ExprEvalStringVector(strings); + return new ExprVectorProcessor() { @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - return (VectorExprEval) eval; + return (ExprEvalVector) eval; } @Override @@ -49,7 +49,7 @@ public ExprType getOutputType() }; } - public static VectorExprProcessor constantDouble(@Nullable Double constant, int maxVectorSize) + public static ExprVectorProcessor constantDouble(@Nullable Double constant, int maxVectorSize) { final double[] doubles = new double[maxVectorSize]; final boolean[] nulls; @@ -60,13 +60,13 @@ public static VectorExprProcessor constantDouble(@Nullable Double constan nulls = null; Arrays.fill(doubles, constant); } - final DoubleVectorExprEval eval = new DoubleVectorExprEval(doubles, nulls); - return new VectorExprProcessor() + final ExprEvalDoubleVector eval = new ExprEvalDoubleVector(doubles, nulls); + return new ExprVectorProcessor() { @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - return (VectorExprEval) eval; + return (ExprEvalVector) eval; } @Override @@ -77,7 +77,7 @@ public ExprType getOutputType() }; } - public static VectorExprProcessor constantLong(@Nullable Long constant, int maxVectorSize) + public static ExprVectorProcessor constantLong(@Nullable Long constant, int maxVectorSize) { final long[] longs = new long[maxVectorSize]; final boolean[] nulls; @@ -88,13 +88,13 @@ public static VectorExprProcessor constantLong(@Nullable Long constant, i nulls = null; Arrays.fill(longs, constant); } - final LongVectorExprEval eval = new LongVectorExprEval(longs, nulls); - return new VectorExprProcessor() + final ExprEvalLongVector eval = new ExprEvalLongVector(longs, nulls); + return new ExprVectorProcessor() { @Override - public VectorExprEval evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - return (VectorExprEval) eval; + return (ExprEvalVector) eval; } @Override @@ -105,9 +105,9 @@ public ExprType getOutputType() }; } - public static VectorExprProcessor parseLong(Expr.VectorInputBindingTypes inputTypes, Expr arg, int radix) + public static ExprVectorProcessor parseLong(Expr.VectorInputBindingTypes inputTypes, Expr arg, int radix) { - final VectorExprProcessor processor = new LongOutStringInFunctionVectorProcessor( + final ExprVectorProcessor processor = new LongOutStringInFunctionVectorProcessor( CastToTypeVectorProcessor.castToType(arg.buildVectorized(inputTypes), ExprType.STRING), inputTypes.getMaxVectorSize() ) @@ -126,6 +126,6 @@ public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, } }; - return (VectorExprProcessor) processor; + return (ExprVectorProcessor) processor; } } diff --git a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java index ea250322ed3e..cc4ecd0415f0 100644 --- a/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java +++ b/core/src/test/java/org/apache/druid/math/expr/VectorExprSanityTest.java @@ -24,7 +24,7 @@ import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.math.expr.vector.VectorExprEval; +import org.apache.druid.math.expr.vector.ExprEvalVector; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.Assert; import org.junit.Test; @@ -174,7 +174,7 @@ private static void testExpressionWithBindings( { Assert.assertTrue(StringUtils.format("Cannot vectorize %s", expr), parsed.canVectorize(bindings.rhs)); ExprType outputType = parsed.getOutputType(bindings.rhs); - VectorExprEval vectorEval = parsed.buildVectorized(bindings.rhs).evalVector(bindings.rhs); + ExprEvalVector vectorEval = parsed.buildVectorized(bindings.rhs).evalVector(bindings.rhs); Assert.assertEquals(outputType, vectorEval.getType()); for (int i = 0; i < VECTOR_SIZE; i++) { ExprEval eval = parsed.eval(bindings.lhs[i]); diff --git a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java index c1555d21e913..7b1fc96779d8 100644 --- a/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java +++ b/processing/src/main/java/org/apache/druid/query/expression/TimestampFloorExprMacro.java @@ -27,8 +27,8 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprType; import org.apache.druid.math.expr.vector.CastToTypeVectorProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.math.expr.vector.LongOutLongInFunctionVectorProcessor; -import org.apache.druid.math.expr.vector.VectorExprProcessor; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -130,9 +130,9 @@ public boolean canVectorize(InputBindingTypes inputTypes) } @Override - public VectorExprProcessor buildVectorized(VectorInputBindingTypes inputTypes) + public ExprVectorProcessor buildVectorized(VectorInputBindingTypes inputTypes) { - VectorExprProcessor processor; + ExprVectorProcessor processor; processor = new LongOutLongInFunctionVectorProcessor( CastToTypeVectorProcessor.castToType(args.get(0).buildVectorized(inputTypes), ExprType.LONG), inputTypes.getMaxVectorSize() @@ -145,7 +145,7 @@ public long apply(long input) } }; - return (VectorExprProcessor) processor; + return (ExprVectorProcessor) processor; } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java index b7346fb0dff3..fde4306261f3 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelector.java @@ -21,15 +21,15 @@ import com.google.common.base.Preconditions; import org.apache.druid.math.expr.Expr; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.segment.vector.VectorObjectSelector; public class ExpressionVectorObjectSelector implements VectorObjectSelector { final Expr.VectorInputBinding bindings; - final VectorExprProcessor processor; + final ExprVectorProcessor processor; - public ExpressionVectorObjectSelector(VectorExprProcessor processor, Expr.VectorInputBinding bindings) + public ExpressionVectorObjectSelector(ExprVectorProcessor processor, Expr.VectorInputBinding bindings) { this.processor = Preconditions.checkNotNull(processor, "processor"); this.bindings = Preconditions.checkNotNull(bindings, "bindings"); diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java index bef524749165..25f2f22758b3 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorSelectors.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprType; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; @@ -46,7 +46,7 @@ public static VectorValueSelector makeVectorValueSelector( final ExpressionPlan plan = ExpressionPlanner.plan(factory, expression); Preconditions.checkArgument(plan.is(ExpressionPlan.Trait.VECTORIZABLE)); final Expr.VectorInputBinding bindings = createVectorBindings(plan.getAnalysis(), factory); - final VectorExprProcessor processor = plan.getExpression().buildVectorized(bindings); + final ExprVectorProcessor processor = plan.getExpression().buildVectorized(bindings); return new ExpressionVectorValueSelector(processor, bindings); } @@ -58,7 +58,7 @@ public static VectorObjectSelector makeVectorObjectSelector( final ExpressionPlan plan = ExpressionPlanner.plan(factory, expression); Preconditions.checkArgument(plan.is(ExpressionPlan.Trait.VECTORIZABLE)); final Expr.VectorInputBinding bindings = createVectorBindings(plan.getAnalysis(), factory); - final VectorExprProcessor processor = plan.getExpression().buildVectorized(bindings); + final ExprVectorProcessor processor = plan.getExpression().buildVectorized(bindings); return new ExpressionVectorObjectSelector(processor, bindings); } diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java index 63bbcf57ebcf..76558c681fc4 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelector.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.math.expr.Expr; -import org.apache.druid.math.expr.vector.VectorExprProcessor; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.apache.druid.segment.vector.VectorValueSelector; import javax.annotation.Nullable; @@ -29,10 +29,10 @@ public class ExpressionVectorValueSelector implements VectorValueSelector { final Expr.VectorInputBinding bindings; - final VectorExprProcessor processor; + final ExprVectorProcessor processor; final float[] floats; - public ExpressionVectorValueSelector(VectorExprProcessor processor, Expr.VectorInputBinding bindings) + public ExpressionVectorValueSelector(ExprVectorProcessor processor, Expr.VectorInputBinding bindings) { this.processor = Preconditions.checkNotNull(processor, "processor"); this.bindings = Preconditions.checkNotNull(bindings, "bindings"); diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index 8e637d9966be..d17abd750ee1 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.math.expr.ExprType; @@ -51,6 +52,8 @@ public class ExpressionVirtualColumn implements VirtualColumn { + private static final Logger log = new Logger(ExpressionVirtualColumn.class); + private final String name; private final String expression; @Nullable @@ -171,6 +174,15 @@ public ColumnCapabilities capabilities(ColumnInspector inspector, String columnN final ExpressionPlan plan = ExpressionPlanner.plan(inspector, parsedExpression.get()); if (plan.getOutputType() != null) { + + if (outputType != null && ExprType.fromValueType(outputType) != plan.getOutputType()) { + log.warn( + "Projected output type %s of expression %s does not match provided type %s", + plan.getOutputType(), + expression, + outputType + ); + } final ExprType inferredOutputType = plan.getOutputType(); final ValueType valueType = ExprType.toValueType(inferredOutputType); if (valueType.isNumeric()) { From 0a51738d7a8f0d37e70cc70c071e4079191fe05a Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 21 Sep 2020 20:21:34 -0700 Subject: [PATCH 14/17] changes --- .../ExpressionVectorSelectorBenchmark.java | 86 +------ .../query/SqlExpressionBenchmark.java | 87 +------ .../java/org/apache/druid/math/expr/Expr.java | 18 +- .../org/apache/druid/math/expr/ExprType.java | 33 ++- .../ColumnSelectorBitmapIndexSelector.java | 10 +- .../segment/QueryableIndexStorageAdapter.java | 13 + .../apache/druid/segment/VirtualColumn.java | 26 +- .../druid/segment/column/RowSignature.java | 19 ++ .../druid/segment/data/ColumnarFloats.java | 2 +- .../virtual/ExpressionVirtualColumn.java | 11 +- .../ExpressionVectorSelectorsTest.java | 240 ++++++++++++++++++ .../druid/sql/calcite/rel/Projection.java | 4 +- .../calcite/rel/VirtualColumnRegistry.java | 5 +- .../SqlVectorizedExpressionSanityTest.java | 233 +++++++++++++++++ 14 files changed, 599 insertions(+), 188 deletions(-) create mode 100644 processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java index a83c686687e9..e969a5f4c93c 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/ExpressionVectorSelectorBenchmark.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.io.Closer; @@ -41,12 +40,11 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.segment.vector.VectorCursor; -import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.segment.virtual.ExpressionVectorSelectorsTest; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; -import org.junit.Assert; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -63,8 +61,6 @@ import org.openjdk.jmh.infra.Blackhole; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.TimeUnit; @@ -218,84 +214,6 @@ public void scan(Blackhole blackhole) private void checkSanity() { - final List results = new ArrayList<>(rowsPerSegment); - final VirtualColumns virtualColumns = VirtualColumns.create( - ImmutableList.of( - new ExpressionVirtualColumn( - "v", - expression, - ExprType.toValueType(outputType), - TestExprMacroTable.INSTANCE - ) - ) - ); - VectorCursor cursor = new QueryableIndexStorageAdapter(index).makeVectorCursor( - null, - index.getDataInterval(), - virtualColumns, - false, - 512, - null - ); - - VectorValueSelector selector = null; - VectorObjectSelector objectSelector = null; - if (outputType.isNumeric()) { - selector = cursor.getColumnSelectorFactory().makeValueSelector("v"); - } else { - objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v"); - } - int rowCount = 0; - while (!cursor.isDone()) { - boolean[] nulls; - switch (outputType) { - case LONG: - nulls = selector.getNullVector(); - long[] longs = selector.getLongVector(); - for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) { - results.add(nulls != null && nulls[i] ? null : longs[i]); - } - break; - case DOUBLE: - nulls = selector.getNullVector(); - double[] doubles = selector.getDoubleVector(); - for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) { - results.add(nulls != null && nulls[i] ? null : doubles[i]); - } - break; - case STRING: - Object[] objects = objectSelector.getObjectVector(); - for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) { - results.add(objects[i]); - } - break; - } - - cursor.advance(); - } - closer.register(cursor); - - Sequence cursors = new QueryableIndexStorageAdapter(index).makeCursors( - null, - index.getDataInterval(), - virtualColumns, - Granularities.ALL, - false, - null - ); - - int rowCountCursor = cursors - .map(nonVectorized -> { - final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory().makeColumnValueSelector("v"); - int rows = 0; - while (!nonVectorized.isDone()) { - Assert.assertEquals(StringUtils.format("Failed at row %s", rows), nonSelector.getObject(), results.get(rows)); - rows++; - nonVectorized.advance(); - } - return rows; - }).accumulate(0, (acc, in) -> acc + in); - - Assert.assertTrue(rowCountCursor > 0); + ExpressionVectorSelectorsTest.sanityTestVectorizedExpressionSelectors(expression, outputType, index, closer, rowsPerSegment); } } diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 8ccfd73a8e39..dba8bd545583 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -23,11 +23,8 @@ import com.google.common.collect.ImmutableMap; import org.apache.calcite.schema.SchemaPlus; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.guava.Yielder; -import org.apache.druid.java.util.common.guava.Yielders; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.DruidProcessingConfig; @@ -40,6 +37,7 @@ import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.DruidPlanner; import org.apache.druid.sql.calcite.planner.PlannerConfig; @@ -49,7 +47,6 @@ import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; -import org.junit.Assert; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -66,7 +63,6 @@ import org.openjdk.jmh.infra.Blackhole; import javax.annotation.Nullable; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -225,7 +221,7 @@ public String getFormatString() private Closer closer = Closer.create(); @Setup(Level.Trial) - public void setup() throws Exception + public void setup() { final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); @@ -267,7 +263,15 @@ public void setup() throws Exception CalciteTests.DRUID_SCHEMA_NAME ); - checkSanity(); + try { + SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries( + plannerFactory, + QUERIES.get(Integer.parseInt(query)) + ); + } + catch (Throwable ignored) { + // the show must go on + } } @TearDown(Level.Trial) @@ -291,73 +295,4 @@ public void querySql(Blackhole blackhole) throws Exception blackhole.consume(lastRow); } } - - public void checkSanity() throws Exception - { - final Map vector = ImmutableMap.of("vectorize", true); - final Map nonvector = ImmutableMap.of("vectorize", false); - final AuthenticationResult authenticationResult = NoopEscalator.getInstance() - .createEscalatedAuthenticationResult(); - - try ( - final DruidPlanner vectorPlanner = plannerFactory.createPlanner(vector, ImmutableList.of(), authenticationResult); - final DruidPlanner nonVectorPlanner = plannerFactory.createPlanner(nonvector, ImmutableList.of(), authenticationResult) - ) { - final PlannerResult vectorPlan = vectorPlanner.plan(QUERIES.get(Integer.parseInt(query))); - final PlannerResult nonVectorPlan = nonVectorPlanner.plan(QUERIES.get(Integer.parseInt(query))); - final Sequence vectorSequence = vectorPlan.run(); - final Sequence nonVectorSequence = nonVectorPlan.run(); - Yielder vectorizedYielder = Yielders.each(vectorSequence); - Yielder nonVectorizedYielder = Yielders.each(nonVectorSequence); - int row = 0; - int misMatch = 0; - while (!vectorizedYielder.isDone() && !nonVectorizedYielder.isDone()) { - Object[] vectorGet = vectorizedYielder.get(); - Object[] nonVectorizedGet = nonVectorizedYielder.get(); - - try { - if (vectorGet[0] instanceof Float || vectorGet[0] instanceof Double) { - Assert.assertArrayEquals( - StringUtils.format( - "Double results differed at row %s (%s : %s)", - row, - Arrays.toString(nonVectorizedGet), - Arrays.toString(vectorGet) - ), - Arrays.stream(nonVectorizedGet).mapToDouble(d -> (double) d).toArray(), - Arrays.stream(vectorGet).mapToDouble(d -> (double) d).toArray(), - 0.01 - ); - } else { - Assert.assertArrayEquals( - StringUtils.format( - "Results differed at row %s (%s : %s)", - row, - Arrays.toString(vectorGet), - Arrays.toString(nonVectorizedGet) - ), - vectorGet, - nonVectorizedGet - ); - } - } - catch (Throwable t) { - log.warn(t.getMessage()); - misMatch++; - } - vectorizedYielder = vectorizedYielder.next(vectorGet); - nonVectorizedYielder = nonVectorizedYielder.next(nonVectorizedGet); - row++; - } - try { - Assert.assertEquals("Expected no mismatched results", 0, misMatch); - Assert.assertTrue(vectorizedYielder.isDone()); - Assert.assertTrue(nonVectorizedYielder.isDone()); - } - catch (Throwable t) { - // the show must go on - log.warn(t.getMessage()); - } - } - } } diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index c7834c2b4d65..81233bc66ff4 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -28,6 +28,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -187,16 +188,7 @@ default boolean areNumeric(List args) default boolean areNumeric(Expr... args) { - boolean numeric = args.length > 0; - for (Expr arg : args) { - ExprType argType = arg.getOutputType(this); - if (argType == null) { - numeric = false; - break; - } - numeric &= argType.isNumeric(); - } - return numeric; + return areNumeric(Arrays.asList(args)); } default boolean canVectorize(List args) @@ -210,11 +202,7 @@ default boolean canVectorize(List args) default boolean canVectorize(Expr... args) { - boolean canVectorize = true; - for (Expr arg : args) { - canVectorize &= arg.canVectorize(this); - } - return canVectorize; + return canVectorize(Arrays.asList(args)); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/ExprType.java b/core/src/main/java/org/apache/druid/math/expr/ExprType.java index c836ef8781e9..05262baf04f5 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ExprType.java +++ b/core/src/main/java/org/apache/druid/math/expr/ExprType.java @@ -50,7 +50,7 @@ public boolean isNumeric() * * @throws IllegalStateException */ - public static ExprType fromValueType(@Nullable ValueType valueType) + public static ExprType fromValueTypeStrict(@Nullable ValueType valueType) { if (valueType == null) { throw new IllegalStateException("Unsupported unknown value type"); @@ -75,6 +75,37 @@ public static ExprType fromValueType(@Nullable ValueType valueType) } } + /** + * The expression system does not distinguish between {@link ValueType#FLOAT} and {@link ValueType#DOUBLE}, and + * cannot currently handle {@link ValueType#COMPLEX} inputs. This method will convert {@link ValueType#FLOAT} to + * {@link #DOUBLE}, or null if a null {@link ValueType#COMPLEX} is encountered. + */ + @Nullable + public static ExprType fromValueType(@Nullable ValueType valueType) + { + if (valueType == null) { + return null; + } + switch (valueType) { + case LONG: + return LONG; + case LONG_ARRAY: + return LONG_ARRAY; + case FLOAT: + case DOUBLE: + return DOUBLE; + case DOUBLE_ARRAY: + return DOUBLE_ARRAY; + case STRING: + return STRING; + case STRING_ARRAY: + return STRING_ARRAY; + case COMPLEX: + default: + return null; + } + } + public static ValueType toValueType(ExprType exprType) { diff --git a/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java index bd6de7a02d57..3eaea9345a6b 100644 --- a/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -161,7 +161,15 @@ public void close() throws IOException public ColumnCapabilities.Capable hasMultipleValues(final String dimension) { if (isVirtualColumn(dimension)) { - return virtualColumns.getVirtualColumn(dimension).capabilities(dimension).hasMultipleValues(); + VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(dimension); + ColumnCapabilities virtualCapabilities = null; + if (virtualColumn != null) { + virtualCapabilities = virtualColumn.capabilities( + QueryableIndexStorageAdapter.getColumnInspectorForIndex(index), + dimension + ); + } + return virtualCapabilities != null ? virtualCapabilities.hasMultipleValues() : ColumnCapabilities.Capable.FALSE; } final ColumnHolder columnHolder = index.getColumnHolder(dimension); diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java index 2ab9e7a536aa..381fcf787630 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexStorageAdapter.java @@ -315,6 +315,19 @@ public static ColumnCapabilities getColumnCapabilities(ColumnSelector index, Str return columnHolder.getCapabilities(); } + public static ColumnInspector getColumnInspectorForIndex(ColumnSelector index) + { + return new ColumnInspector() + { + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + return QueryableIndexStorageAdapter.getColumnCapabilities(index, column); + } + }; + } + @Override public Metadata getMetadata() { diff --git a/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java index c4b86518856e..3193ad58b146 100644 --- a/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumn.java @@ -235,18 +235,34 @@ default VectorObjectSelector makeVectorObjectSelector( } /** - * Returns the capabilities of this virtual column, which includes a type that corresponds to the best - * performing base selector supertype (e. g. {@link BaseLongColumnValueSelector}) of the object, returned from - * {@link #makeColumnValueSelector(String, ColumnSelectorFactory)}. May vary based on columnName if this column uses - * dot notation. + * This method is deprecated in favor of {@link #capabilities(ColumnInspector, String)}, which should be used whenever + * possible and can support virtual column implementations that need to inspect other columns as inputs. + * + * This is a fallback implementation to return the capabilities of this virtual column, which includes a type that + * corresponds to the best performing base selector supertype (e. g. {@link BaseLongColumnValueSelector}) of the + * object, returned from {@link #makeColumnValueSelector(String, ColumnSelectorFactory)}. May vary based on columnName + * if this column uses dot notation. * * @param columnName the name this virtual column was referenced with * * @return capabilities, must not be null */ + @Deprecated ColumnCapabilities capabilities(String columnName); - + /** + * Return the {@link ColumnCapabilities} which best describe the optimal selector to read from this virtual column. + * + * The {@link ColumnInspector} (most likely corresponding to an underlying {@link ColumnSelectorFactory} of a query) + * allows the virtual column to consider this information if necessary to compute its output type details. + * + * Examples of this include the {@link ExpressionVirtualColumn}, which takes input from other columns and uses the + * {@link ColumnInspector} to infer the output type of expressions based on the types of the inputs. + * + * @param inspector column inspector to provide additional information of other available columns + * @param columnName the name this virtual column was referenced with + * @return capabilities, must not be null + */ default ColumnCapabilities capabilities(ColumnInspector inspector, String columnName) { return capabilities(columnName); diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index 0b00edb0e944..581b6b29870d 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -28,6 +28,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.segment.ColumnInspector; import javax.annotation.Nullable; import java.util.ArrayList; @@ -157,6 +158,24 @@ public int indexOf(final String columnName) return columnPositions.applyAsInt(columnName); } + public ColumnInspector asColumnInspector() + { + return new ColumnInspector() + { + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + return getColumnType(column).map(valueType -> { + if (valueType.isNumeric()) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); + } + return new ColumnCapabilitiesImpl().setType(valueType); + }).orElse(null); + } + }; + } + @Override public boolean equals(Object o) { diff --git a/processing/src/main/java/org/apache/druid/segment/data/ColumnarFloats.java b/processing/src/main/java/org/apache/druid/segment/data/ColumnarFloats.java index b9764f59f2b2..5e4d1ed7bdab 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/ColumnarFloats.java +++ b/processing/src/main/java/org/apache/druid/segment/data/ColumnarFloats.java @@ -81,7 +81,7 @@ public float getFloat() @Override public double getDouble(int offset) { - return ColumnarFloats.this.get(offset); + return (double) ColumnarFloats.this.get(offset); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java index d17abd750ee1..8ad46d6436ec 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ExpressionVirtualColumn.java @@ -185,19 +185,26 @@ public ColumnCapabilities capabilities(ColumnInspector inspector, String columnN } final ExprType inferredOutputType = plan.getOutputType(); final ValueType valueType = ExprType.toValueType(inferredOutputType); + if (valueType.isNumeric()) { // if float was explicitly specified preserve it, because it will currently never be the computed output type if (ValueType.FLOAT == outputType) { return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ValueType.FLOAT); } - return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(ExprType.toValueType(inferredOutputType)); + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); + } + + // null constants can sometimes trip up the type inference to report STRING, so check if explicitly supplied + // output type is numeric and stick with that if so + if (outputType != null && outputType.isNumeric()) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(outputType); } // we don't have to check for unknown input here because output type is unable to be inferred if we don't know // the complete set of input types if (plan.any(ExpressionPlan.Trait.NON_SCALAR_OUTPUT, ExpressionPlan.Trait.NEEDS_APPLIED)) { // always a multi-value string since wider engine does not yet support array types - return new ColumnCapabilitiesImpl().setType(ValueType.STRING); + return new ColumnCapabilitiesImpl().setType(ValueType.STRING).setHasMultipleValues(true); } // if we got here, lets call it single value string output diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java new file mode 100644 index 000000000000..9c7926fe9bbc --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.math.expr.ExprType; +import org.apache.druid.math.expr.Parser; +import org.apache.druid.query.expression.TestExprMacroTable; +import org.apache.druid.segment.ColumnInspector; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.Cursor; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.QueryableIndexStorageAdapter; +import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.generator.GeneratorBasicSchemas; +import org.apache.druid.segment.generator.GeneratorSchemaInfo; +import org.apache.druid.segment.generator.SegmentGenerator; +import org.apache.druid.segment.vector.VectorCursor; +import org.apache.druid.segment.vector.VectorObjectSelector; +import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@RunWith(Parameterized.class) +public class ExpressionVectorSelectorsTest +{ + private static List EXPRESSIONS = ImmutableList.of( + "long1 * long2", + "double1 * double3", + "float1 + float3", + "(long1 - long4) / double3", + "long5 * float3 * long1 * long4 * double1", + "long5 * double3 * long1 * long4 * double1", + "max(double3, double5)", + "min(double4, double1)", + "cos(float3)", + "sin(long4)", + "parse_long(string1)", + "parse_long(string1) * double3", + "parse_long(string5) * parse_long(string1)", + "parse_long(string5) * parse_long(string1) * double3" + ); + + private static final int ROWS_PER_SEGMENT = 100_000; + + private static QueryableIndex INDEX; + private static Closer CLOSER; + + @BeforeClass + public static void setupClass() + { + CLOSER = Closer.create(); + + final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); + + final DataSegment dataSegment = DataSegment.builder() + .dataSource("foo") + .interval(schemaInfo.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(); + + final SegmentGenerator segmentGenerator = CLOSER.register(new SegmentGenerator()); + INDEX = CLOSER.register( + segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, ROWS_PER_SEGMENT) + ); + } + + @AfterClass + public static void teardownClass() throws IOException + { + CLOSER.close(); + } + + @Parameterized.Parameters(name = "expression = {0}") + public static Iterable constructorFeeder() + { + return EXPRESSIONS.stream().map(x -> new Object[]{x}).collect(Collectors.toList()); + } + + @Nullable + private ExprType outputType; + private String expression; + + public ExpressionVectorSelectorsTest(String expression) + { + this.expression = expression; + } + + @Before + public void setup() + { + Expr parsed = Parser.parse(expression, ExprMacroTable.nil()); + outputType = parsed.getOutputType( + new ColumnInspector() + { + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + return QueryableIndexStorageAdapter.getColumnCapabilities(INDEX, column); + } + } + ); + } + + @Test + public void sanityTestVectorizedExpressionSelector() + { + sanityTestVectorizedExpressionSelectors(expression, outputType, INDEX, CLOSER, ROWS_PER_SEGMENT); + } + + public static void sanityTestVectorizedExpressionSelectors( + String expression, + @Nullable ExprType outputType, + QueryableIndex index, + Closer closer, + int rowsPerSegment + ) + { + final List results = new ArrayList<>(rowsPerSegment); + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn( + "v", + expression, + ExprType.toValueType(outputType), + TestExprMacroTable.INSTANCE + ) + ) + ); + VectorCursor cursor = new QueryableIndexStorageAdapter(index).makeVectorCursor( + null, + index.getDataInterval(), + virtualColumns, + false, + 512, + null + ); + + VectorValueSelector selector = null; + VectorObjectSelector objectSelector = null; + if (outputType.isNumeric()) { + selector = cursor.getColumnSelectorFactory().makeValueSelector("v"); + } else { + objectSelector = cursor.getColumnSelectorFactory().makeObjectSelector("v"); + } + int rowCount = 0; + while (!cursor.isDone()) { + boolean[] nulls; + switch (outputType) { + case LONG: + nulls = selector.getNullVector(); + long[] longs = selector.getLongVector(); + for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) { + results.add(nulls != null && nulls[i] ? null : longs[i]); + } + break; + case DOUBLE: + nulls = selector.getNullVector(); + double[] doubles = selector.getDoubleVector(); + for (int i = 0; i < selector.getCurrentVectorSize(); i++, rowCount++) { + results.add(nulls != null && nulls[i] ? null : doubles[i]); + } + break; + case STRING: + Object[] objects = objectSelector.getObjectVector(); + for (int i = 0; i < objectSelector.getCurrentVectorSize(); i++, rowCount++) { + results.add(objects[i]); + } + break; + } + + cursor.advance(); + } + closer.register(cursor); + + Sequence cursors = new QueryableIndexStorageAdapter(index).makeCursors( + null, + index.getDataInterval(), + virtualColumns, + Granularities.ALL, + false, + null + ); + + int rowCountCursor = cursors + .map(nonVectorized -> { + final ColumnValueSelector nonSelector = nonVectorized.getColumnSelectorFactory().makeColumnValueSelector("v"); + int rows = 0; + while (!nonVectorized.isDone()) { + Assert.assertEquals(StringUtils.format("Failed at row %s", rows), nonSelector.getObject(), results.get(rows)); + rows++; + nonVectorized.advance(); + } + return rows; + }).accumulate(0, (acc, in) -> acc + in); + + Assert.assertTrue(rowCountCursor > 0); + Assert.assertEquals(rowCountCursor, rowCount); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java index 02cc3ce2aae3..631b338f6816 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/Projection.java @@ -316,8 +316,8 @@ private static boolean postAggregatorDirectColumnIsOk( } // Check if a cast is necessary. - final ExprType toExprType = ExprType.fromValueType(columnValueType); - final ExprType fromExprType = ExprType.fromValueType( + final ExprType toExprType = ExprType.fromValueTypeStrict(columnValueType); + final ExprType fromExprType = ExprType.fromValueTypeStrict( Calcites.getValueTypeForRelDataType(rexNode.getType()) ); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java index d6ff84003037..c2664957c5f9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.rel; import org.apache.calcite.rel.type.RelDataType; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; @@ -136,9 +137,11 @@ public RowSignature getFullRowSignature() final RowSignature.Builder builder = RowSignature.builder().addAll(baseRowSignature); + ColumnInspector baseRowsInspector = builder.build().asColumnInspector(); + for (VirtualColumn virtualColumn : virtualColumnsByName.values()) { final String columnName = virtualColumn.getOutputName(); - builder.add(columnName, virtualColumn.capabilities(columnName).getType()); + builder.add(columnName, virtualColumn.capabilities(baseRowsInspector, columnName).getType()); } return builder.build(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java new file mode 100644 index 000000000000..10d630bff3c2 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.tools.RelConversionException; +import org.apache.calcite.tools.ValidationException; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.generator.GeneratorBasicSchemas; +import org.apache.druid.segment.generator.GeneratorSchemaInfo; +import org.apache.druid.segment.generator.SegmentGenerator; +import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.security.AuthTestUtils; +import org.apache.druid.server.security.AuthenticationResult; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.planner.DruidPlanner; +import org.apache.druid.sql.calcite.planner.PlannerConfig; +import org.apache.druid.sql.calcite.planner.PlannerFactory; +import org.apache.druid.sql.calcite.planner.PlannerResult; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@RunWith(Parameterized.class) +public class SqlVectorizedExpressionSanityTest extends InitializedNullHandlingTest +{ + private static final Logger log = new Logger(SqlVectorizedExpressionSanityTest.class); + + private static final List QUERIES = ImmutableList.of( + "SELECT SUM(long1 * long2) FROM foo", + "SELECT SUM((long1 * long2) / double1) FROM foo", + "SELECT SUM(float3 + ((long1 * long4)/double1)) FROM foo", + "SELECT SUM(long5 - (float3 + ((long1 * long4)/double1))) FROM foo", + "SELECT cos(double2) FROM foo", + "SELECT SUM(-long4) FROM foo", + "SELECT SUM(PARSE_LONG(string1)) FROM foo", + "SELECT SUM(PARSE_LONG(string3)) FROM foo", + "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3", + "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 1", + "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2", + "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 1", + "SELECT (long1 * long2), SUM(double1) FROM foo GROUP BY 1 ORDER BY 2", + "SELECT string2, SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2" + ); + + private static final int ROWS_PER_SEGMENT = 100_000; + + private static QueryableIndex INDEX; + private static Closer CLOSER; + private static QueryRunnerFactoryConglomerate CONGLOMERATE; + private static SpecificSegmentsQuerySegmentWalker WALKER; + @Nullable + private static PlannerFactory PLANNER_FACTORY; + + @BeforeClass + public static void setupClass() + { + CLOSER = Closer.create(); + + final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench"); + + final DataSegment dataSegment = DataSegment.builder() + .dataSource("foo") + .interval(schemaInfo.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(); + + final SegmentGenerator segmentGenerator = CLOSER.register(new SegmentGenerator()); + INDEX = CLOSER.register( + segmentGenerator.generate(dataSegment, schemaInfo, Granularities.HOUR, ROWS_PER_SEGMENT) + ); + CONGLOMERATE = QueryStackTests.createQueryRunnerFactoryConglomerate(CLOSER); + + WALKER = new SpecificSegmentsQuerySegmentWalker(CONGLOMERATE).add( + dataSegment, + INDEX + ); + CLOSER.register(WALKER); + + final PlannerConfig plannerConfig = new PlannerConfig(); + final SchemaPlus rootSchema = + CalciteTests.createMockRootSchema(CONGLOMERATE, WALKER, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER); + PLANNER_FACTORY = new PlannerFactory( + rootSchema, + CalciteTests.createMockQueryLifecycleFactory(WALKER, CONGLOMERATE), + CalciteTests.createOperatorTable(), + CalciteTests.createExprMacroTable(), + plannerConfig, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + CalciteTests.getJsonMapper(), + CalciteTests.DRUID_SCHEMA_NAME + ); + } + + @AfterClass + public static void teardownClass() throws IOException + { + CLOSER.close(); + } + + @Parameterized.Parameters(name = "query = {0}") + public static Iterable constructorFeeder() + { + return QUERIES.stream().map(x -> new Object[]{x}).collect(Collectors.toList()); + } + + private String query; + + public SqlVectorizedExpressionSanityTest(String query) + { + this.query = query; + } + + @Test + public void testQuery() throws SqlParseException, RelConversionException, ValidationException + { + sanityTestVectorizedSqlQueries(PLANNER_FACTORY, query); + } + + + public static void sanityTestVectorizedSqlQueries(PlannerFactory plannerFactory, String query) + throws ValidationException, RelConversionException, SqlParseException + { + final Map vector = ImmutableMap.of("vectorize", true); + final Map nonvector = ImmutableMap.of("vectorize", false); + final AuthenticationResult authenticationResult = NoopEscalator.getInstance() + .createEscalatedAuthenticationResult(); + + try ( + final DruidPlanner vectorPlanner = plannerFactory.createPlanner(vector, ImmutableList.of(), authenticationResult); + final DruidPlanner nonVectorPlanner = plannerFactory.createPlanner(nonvector, ImmutableList.of(), authenticationResult) + ) { + final PlannerResult vectorPlan = vectorPlanner.plan(query); + final PlannerResult nonVectorPlan = nonVectorPlanner.plan(query); + final Sequence vectorSequence = vectorPlan.run(); + final Sequence nonVectorSequence = nonVectorPlan.run(); + Yielder vectorizedYielder = Yielders.each(vectorSequence); + Yielder nonVectorizedYielder = Yielders.each(nonVectorSequence); + int row = 0; + int misMatch = 0; + while (!vectorizedYielder.isDone() && !nonVectorizedYielder.isDone()) { + Object[] vectorGet = vectorizedYielder.get(); + Object[] nonVectorizedGet = nonVectorizedYielder.get(); + + try { + Assert.assertEquals(vectorGet.length, nonVectorizedGet.length); + for (int i = 0; i < vectorGet.length; i++) { + Object nonVectorObject = nonVectorizedGet[i]; + Object vectorObject = vectorGet[i]; + if (vectorObject instanceof Float || vectorObject instanceof Double) { + Assert.assertEquals( + StringUtils.format( + "Double results differed at row %s (%s : %s)", + row, + nonVectorObject, + vectorObject + ), + ((Double) nonVectorObject).doubleValue(), + ((Double) vectorObject).doubleValue(), + 0.01 + ); + } else { + Assert.assertEquals( + StringUtils.format( + "Results differed at row %s (%s : %s)", + row, + nonVectorObject, + vectorObject + ), + nonVectorObject, + vectorObject + ); + } + } + } + catch (Throwable t) { + log.warn(t.getMessage()); + misMatch++; + } + vectorizedYielder = vectorizedYielder.next(vectorGet); + nonVectorizedYielder = nonVectorizedYielder.next(nonVectorizedGet); + row++; + } + Assert.assertEquals("Expected no mismatched results", 0, misMatch); + Assert.assertTrue(vectorizedYielder.isDone()); + Assert.assertTrue(nonVectorizedYielder.isDone()); + } + } +} From 774373656e353e4edba6708dde192210320975e5 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 21 Sep 2020 20:26:27 -0700 Subject: [PATCH 15/17] nullable --- .../org/apache/druid/query/aggregation/AggregatorUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index dfa6ad051b30..bef668db4bbb 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -231,8 +231,8 @@ public boolean isNull() public static VectorValueSelector makeVectorValueSelector( VectorColumnSelectorFactory columnSelectorFactory, - String fieldName, - String expression, + @Nullable String fieldName, + @Nullable String expression, Supplier fieldExpression ) { From cf325119f9612fefb302fe9ff72f5f5b365e3522 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 21 Sep 2020 20:28:23 -0700 Subject: [PATCH 16/17] missing hex --- .../apache/druid/math/expr/vector/VectorProcessors.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index 11aced2ce6ac..9589aa5a3ebe 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -116,7 +116,13 @@ public static ExprVectorProcessor parseLong(Expr.VectorInputBindingTypes public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) { try { - longs[i] = Long.parseLong(strings[i], radix); + final String input = strings[i]; + if (radix == 16 && (input.startsWith("0x") || input.startsWith("0X"))) { + // Strip leading 0x from hex strings. + longs[i] = Long.parseLong(input.substring(2), radix); + } else { + longs[i] = Long.parseLong(input, radix); + } outputNulls[i] = false; } catch (NumberFormatException e) { From 4fa4f299aeac07ab1b1f0c3bc51f84798c2305a0 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 22 Sep 2020 16:51:12 -0700 Subject: [PATCH 17/17] more --- .../apache/druid/math/expr/ApplyFunction.java | 15 +++++ .../math/expr/BinaryLogicalOperatorExpr.java | 1 + .../java/org/apache/druid/math/expr/Expr.java | 16 ++++++ .../org/apache/druid/math/expr/Function.java | 18 +++++- .../vector/VectorComparisonProcessors.java | 10 ++-- .../math/expr/vector/VectorProcessors.java | 5 ++ .../query/aggregation/AggregatorUtil.java | 55 +++++++++++++------ .../SimpleDoubleAggregatorFactory.java | 9 +-- .../SimpleFloatAggregatorFactory.java | 9 +-- .../SimpleLongAggregatorFactory.java | 9 +-- .../druid/segment/column/RowSignature.java | 32 +++++------ .../calcite/rel/VirtualColumnRegistry.java | 5 +- 12 files changed, 116 insertions(+), 68 deletions(-) diff --git a/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java b/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java index 73fe42153172..891216c9c6b8 100644 --- a/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java +++ b/core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java @@ -51,11 +51,26 @@ public interface ApplyFunction */ String name(); + /** + * Check if an apply function can be 'vectorized', for a given {@link LambdaExpr} and set of {@link Expr} inputs. + * If this method returns true, {@link #asVectorProcessor} is expected to produce a {@link ExprVectorProcessor} which + * can evaluate values in batches to use with vectorized query engines. + * + * @see Expr#canVectorize(Expr.InputBindingTypes) + * @see Function#canVectorize(Expr.InputBindingTypes, List) + */ default boolean canVectorize(Expr.InputBindingTypes inputTypes, Expr lambda, List args) { return false; } + /** + * Builds a 'vectorized' function expression processor, that can build vectorized processors for its input values + * using {@link Expr#buildVectorized}, for use in vectorized query engines. + * + * @see Expr#buildVectorized(Expr.VectorInputBindingTypes) + * @see Function#asVectorProcessor(Expr.VectorInputBindingTypes, List) + */ default ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, Expr lambda, List args) { throw new UOE("%s is not vectorized", name()); diff --git a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java index e0a34892ee5b..a230cd638609 100644 --- a/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java @@ -70,6 +70,7 @@ public ExprType getOutputType(InputBindingTypes inputTypes) } return implicitCast; } + @Override public boolean canVectorize(InputBindingTypes inputTypes) { diff --git a/core/src/main/java/org/apache/druid/math/expr/Expr.java b/core/src/main/java/org/apache/druid/math/expr/Expr.java index 81233bc66ff4..b8fa44f6a242 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Expr.java +++ b/core/src/main/java/org/apache/druid/math/expr/Expr.java @@ -172,6 +172,11 @@ interface InputBindingTypes @Nullable ExprType getType(String name); + /** + * Check if all provided {@link Expr} can infer the output type as {@link ExprType#isNumeric} with a value of true. + * + * There must be at least one expression with a computable numeric output type for this method to return true. + */ default boolean areNumeric(List args) { boolean numeric = args.size() > 0; @@ -186,11 +191,19 @@ default boolean areNumeric(List args) return numeric; } + /** + * Check if all provided {@link Expr} can infer the output type as {@link ExprType#isNumeric} with a value of true. + * + * There must be at least one expression with a computable numeric output type for this method to return true. + */ default boolean areNumeric(Expr... args) { return areNumeric(Arrays.asList(args)); } + /** + * Check if every provided {@link Expr} computes {@link Expr#canVectorize(InputBindingTypes)} to a value of true + */ default boolean canVectorize(List args) { boolean canVectorize = true; @@ -200,6 +213,9 @@ default boolean canVectorize(List args) return canVectorize; } + /** + * Check if every provided {@link Expr} computes {@link Expr#canVectorize(InputBindingTypes)} to a value of true + */ default boolean canVectorize(Expr... args) { return canVectorize(Arrays.asList(args)); diff --git a/core/src/main/java/org/apache/druid/math/expr/Function.java b/core/src/main/java/org/apache/druid/math/expr/Function.java index afdd812e06b6..4fde3398d501 100644 --- a/core/src/main/java/org/apache/druid/math/expr/Function.java +++ b/core/src/main/java/org/apache/druid/math/expr/Function.java @@ -117,11 +117,26 @@ default boolean hasArrayOutput() @Nullable ExprType getOutputType(Expr.InputBindingTypes inputTypes, List args); + /** + * Check if a function can be 'vectorized', for a given set of {@link Expr} inputs. If this method returns true, + * {@link #asVectorProcessor} is expected to produce a {@link ExprVectorProcessor} which can evaluate values in + * batches to use with vectorized query engines. + * + * @see Expr#canVectorize(Expr.InputBindingTypes) + * @see ApplyFunction#canVectorize(Expr.InputBindingTypes, Expr, List) + */ default boolean canVectorize(Expr.InputBindingTypes inputTypes, List args) { return false; } + /** + * Builds a 'vectorized' function expression processor, that can build vectorized processors for its input values + * using {@link Expr#buildVectorized}, for use in vectorized query engines. + * + * @see Expr#buildVectorized(Expr.VectorInputBindingTypes) + * @see ApplyFunction#asVectorProcessor(Expr.VectorInputBindingTypes, Expr, List) + */ default ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, List args) { throw new UOE("%s is not vectorized", name()); @@ -547,7 +562,8 @@ public ExprVectorProcessor asVectorProcessor(Expr.VectorInputBindingTypes final int radix = args.size() == 1 ? 10 : ((Number) args.get(1).getLiteralValue()).intValue(); return VectorProcessors.parseLong(inputTypes, args.get(0), radix); } - // not yet implemented, how did we get here + // only single argument and 2 argument where the radix is constant is currently implemented + // the canVectorize check should prevent this from happening, but explode just in case throw Exprs.cannotVectorize(this); } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java index 49abf5bbf5a2..972d1d62e07c 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java @@ -24,11 +24,6 @@ public class VectorComparisonProcessors { - private VectorComparisonProcessors() - { - // No instantiation - } - public static ExprVectorProcessor equal( Expr.VectorInputBindingTypes inputTypes, Expr left, @@ -394,4 +389,9 @@ public double apply(double left, double right) } ); } + + private VectorComparisonProcessors() + { + // No instantiation + } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index 9589aa5a3ebe..9ae7ab7c8898 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -134,4 +134,9 @@ public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, return (ExprVectorProcessor) processor; } + + private VectorProcessors() + { + // No instantiation + } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java index bef668db4bbb..2cc5f1b06662 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/AggregatorUtil.java @@ -26,11 +26,14 @@ import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprEval; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DoubleColumnSelector; import org.apache.druid.segment.FloatColumnSelector; import org.apache.druid.segment.LongColumnSelector; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.vector.VectorColumnSelectorFactory; import org.apache.druid.segment.vector.VectorValueSelector; import org.apache.druid.segment.virtual.ExpressionSelectors; @@ -229,22 +232,6 @@ public boolean isNull() } } - public static VectorValueSelector makeVectorValueSelector( - VectorColumnSelectorFactory columnSelectorFactory, - @Nullable String fieldName, - @Nullable String expression, - Supplier fieldExpression - ) - { - if ((fieldName == null) == (expression == null)) { - throw new IllegalArgumentException("Only one of fieldName or expression should be non-null"); - } - if (expression != null) { - return ExpressionVectorSelectors.makeVectorValueSelector(columnSelectorFactory, fieldExpression.get()); - } - return columnSelectorFactory.makeValueSelector(fieldName); - } - /** * Only one of fieldName and fieldExpression should be non-null */ @@ -330,4 +317,40 @@ public boolean isNull() return new ExpressionDoubleColumnSelector(); } } + + public static boolean canVectorize( + ColumnInspector columnInspector, + @Nullable String fieldName, + @Nullable String expression, + Supplier fieldExpression + ) + { + if (fieldName != null) { + final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); + return capabilities == null || ValueType.isNumeric(capabilities.getType()); + } + if (expression != null) { + return fieldExpression.get().canVectorize(columnInspector); + } + return false; + } + + /** + * Make a {@link VectorValueSelector} for primitive numeric or expression virtual column inputs. + */ + public static VectorValueSelector makeVectorValueSelector( + VectorColumnSelectorFactory columnSelectorFactory, + @Nullable String fieldName, + @Nullable String expression, + Supplier fieldExpression + ) + { + if ((fieldName == null) == (expression == null)) { + throw new IllegalArgumentException("Only one of fieldName or expression should be non-null"); + } + if (expression != null) { + return ExpressionVectorSelectors.makeVectorValueSelector(columnSelectorFactory, fieldExpression.get()); + } + return columnSelectorFactory.makeValueSelector(fieldName); + } } diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java index 41c07fd4a4e3..b8644329c71f 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleDoubleAggregatorFactory.java @@ -247,14 +247,7 @@ public String getExpression() @Override public boolean canVectorize(ColumnInspector columnInspector) { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return capabilities == null || ValueType.isNumeric(capabilities.getType()); - } - if (expression != null) { - return fieldExpression.get().canVectorize(columnInspector); - } - return false; + return AggregatorUtil.canVectorize(columnInspector, fieldName, expression, fieldExpression); } protected abstract double nullValue(); diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java index 1f41f4ce2213..380ceb149419 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleFloatAggregatorFactory.java @@ -226,14 +226,7 @@ public String getExpression() @Override public boolean canVectorize(ColumnInspector columnInspector) { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return capabilities == null || ValueType.isNumeric(capabilities.getType()); - } - if (expression != null) { - return fieldExpression.get().canVectorize(columnInspector); - } - return false; + return AggregatorUtil.canVectorize(columnInspector, fieldName, expression, fieldExpression); } private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory) diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java index ecacbf836b68..7d148d5b6f0b 100644 --- a/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java +++ b/processing/src/main/java/org/apache/druid/query/aggregation/SimpleLongAggregatorFactory.java @@ -229,14 +229,7 @@ public String getExpression() @Override public boolean canVectorize(ColumnInspector columnInspector) { - if (fieldName != null) { - final ColumnCapabilities capabilities = columnInspector.getColumnCapabilities(fieldName); - return capabilities == null || ValueType.isNumeric(capabilities.getType()); - } - if (expression != null) { - return fieldExpression.get().canVectorize(columnInspector); - } - return false; + return AggregatorUtil.canVectorize(columnInspector, fieldName, expression, fieldExpression); } private boolean shouldUseStringColumnAggregatorWrapper(ColumnSelectorFactory columnSelectorFactory) diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index 581b6b29870d..c988f08acc96 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -47,7 +47,7 @@ * @see org.apache.druid.query.QueryToolChest#resultArraySignature which returns signatures for query results * @see org.apache.druid.query.InlineDataSource#getRowSignature which returns signatures for inline datasources */ -public class RowSignature +public class RowSignature implements ColumnInspector { private static final RowSignature EMPTY = new RowSignature(Collections.emptyList()); @@ -158,24 +158,6 @@ public int indexOf(final String columnName) return columnPositions.applyAsInt(columnName); } - public ColumnInspector asColumnInspector() - { - return new ColumnInspector() - { - @Nullable - @Override - public ColumnCapabilities getColumnCapabilities(String column) - { - return getColumnType(column).map(valueType -> { - if (valueType.isNumeric()) { - return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); - } - return new ColumnCapabilitiesImpl().setType(valueType); - }).orElse(null); - } - }; - } - @Override public boolean equals(Object o) { @@ -210,6 +192,18 @@ public String toString() return s.append("}").toString(); } + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String column) + { + return getColumnType(column).map(valueType -> { + if (valueType.isNumeric()) { + return ColumnCapabilitiesImpl.createSimpleNumericColumnCapabilities(valueType); + } + return new ColumnCapabilitiesImpl().setType(valueType); + }).orElse(null); + } + public static class Builder { private final List> columnTypeList; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java index c2664957c5f9..7244c750574a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/VirtualColumnRegistry.java @@ -20,7 +20,6 @@ package org.apache.druid.sql.calcite.rel; import org.apache.calcite.rel.type.RelDataType; -import org.apache.druid.segment.ColumnInspector; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; @@ -137,11 +136,11 @@ public RowSignature getFullRowSignature() final RowSignature.Builder builder = RowSignature.builder().addAll(baseRowSignature); - ColumnInspector baseRowsInspector = builder.build().asColumnInspector(); + RowSignature baseSignature = builder.build(); for (VirtualColumn virtualColumn : virtualColumnsByName.values()) { final String columnName = virtualColumn.getOutputName(); - builder.add(columnName, virtualColumn.capabilities(baseRowsInspector, columnName).getType()); + builder.add(columnName, virtualColumn.capabilities(baseSignature, columnName).getType()); } return builder.build();