Skip to content

support for vectorizing expressions with non-existent inputs, more consistent type handling for non-vectorized expressions #10499

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
clintropolis merged 6 commits into from
Oct 27, 2020
Merged

support for vectorizing expressions with non-existent inputs, more consistent type handling for non-vectorized expressions #10499

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d3c3347
support for vectorizing expressions with non-existent inputs, more co…
clintropolis Oct 9, 2020
24a0beb
inspector
clintropolis Oct 9, 2020
a537344
Merge remote-tracking branch 'upstream/master' into vector-nil-expr
clintropolis Oct 23, 2020
5acd0f0
changes
clintropolis Oct 23, 2020
06b233c
more test
clintropolis Oct 23, 2020
6a58eb3
clean
clintropolis Oct 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 26 additions & 22 deletions core/src/main/java/org/apache/druid/math/expr/ApplyFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ public interface ApplyFunction
* If this method returns true, {@link #asVectorProcessor} is expected to produce a {@link ExprVectorProcessor} which
* can evaluate values in batches to use with vectorized query engines.
*
* @see Expr#canVectorize(Expr.InputBindingTypes)
* @see Function#canVectorize(Expr.InputBindingTypes, List)
* @see Expr#canVectorize(Expr.InputBindingInspector)
* @see Function#canVectorize(Expr.InputBindingInspector, List)
*/
default boolean canVectorize(Expr.InputBindingTypes inputTypes, Expr lambda, List<Expr> args)
default boolean canVectorize(Expr.InputBindingInspector inspector, Expr lambda, List<Expr> args)
{
return false;
}
Expand All @@ -68,10 +68,14 @@ default boolean canVectorize(Expr.InputBindingTypes inputTypes, Expr lambda, Lis
* Builds a 'vectorized' function expression processor, that can build vectorized processors for its input values
* using {@link Expr#buildVectorized}, for use in vectorized query engines.
*
* @see Expr#buildVectorized(Expr.VectorInputBindingTypes)
* @see Function#asVectorProcessor(Expr.VectorInputBindingTypes, List)
* @see Expr#buildVectorized(Expr.VectorInputBindingInspector)
* @see Function#asVectorProcessor(Expr.VectorInputBindingInspector, List)
*/
default <T> ExprVectorProcessor<T> asVectorProcessor(Expr.VectorInputBindingTypes inputTypes, Expr lambda, List<Expr> args)
default <T> ExprVectorProcessor<T> asVectorProcessor(
Expr.VectorInputBindingInspector inspector,
Expr lambda,
List<Expr> args
)
{
throw new UOE("%s is not vectorized", name());
}
Expand Down Expand Up @@ -109,7 +113,7 @@ default boolean hasArrayOutput(LambdaExpr lambdaExpr)
* @see Expr#getOutputType
*/
@Nullable
ExprType getOutputType(Expr.InputBindingTypes inputTypes, LambdaExpr expr, List<Expr> args);
ExprType getOutputType(Expr.InputBindingInspector inspector, LambdaExpr expr, List<Expr> args);

/**
* Base class for "map" functions, which are a class of {@link ApplyFunction} which take a lambda function that is
Expand All @@ -126,9 +130,9 @@ public boolean hasArrayOutput(LambdaExpr lambdaExpr)

@Nullable
@Override
public ExprType getOutputType(Expr.InputBindingTypes inputTypes, LambdaExpr expr, List<Expr> args)
public ExprType getOutputType(Expr.InputBindingInspector inspector, LambdaExpr expr, List<Expr> args)
{
return ExprType.asArrayType(expr.getOutputType(new LambdaInputBindingTypes(inputTypes, expr, args)));
return ExprType.asArrayType(expr.getOutputType(new LambdaInputBindingInspector(inspector, expr, args)));
}

/**
Expand Down Expand Up @@ -332,10 +336,10 @@ public boolean hasArrayOutput(LambdaExpr lambdaExpr)

@Nullable
@Override
public ExprType getOutputType(Expr.InputBindingTypes inputTypes, LambdaExpr expr, List<Expr> args)
public ExprType getOutputType(Expr.InputBindingInspector inspector, LambdaExpr expr, List<Expr> args)
{
// output type is accumulator type, which is last argument
return args.get(args.size() - 1).getOutputType(inputTypes);
return args.get(args.size() - 1).getOutputType(inspector);
}
}

Expand Down Expand Up @@ -535,10 +539,10 @@ public void validateArguments(LambdaExpr lambdaExpr, List<Expr> args)

@Nullable
@Override
public ExprType getOutputType(Expr.InputBindingTypes inputTypes, LambdaExpr expr, List<Expr> args)
public ExprType getOutputType(Expr.InputBindingInspector inspector, LambdaExpr expr, List<Expr> args)
{
// output type is input array type
return args.get(0).getOutputType(inputTypes);
return args.get(0).getOutputType(inspector);
}

private <T> Stream<T> filter(T[] array, LambdaExpr expr, SettableLambdaBinding binding)
Expand Down Expand Up @@ -590,7 +594,7 @@ public void validateArguments(LambdaExpr lambdaExpr, List<Expr> args)

@Nullable
@Override
public ExprType getOutputType(Expr.InputBindingTypes inputTypes, LambdaExpr expr, List<Expr> args)
public ExprType getOutputType(Expr.InputBindingInspector inspector, LambdaExpr expr, List<Expr> args)
{
return ExprType.LONG;
}
Expand Down Expand Up @@ -917,20 +921,20 @@ public CartesianFoldLambdaBinding accumulateWithIndex(int index, Object acc)
}

/**
* Helper that can wrap another {@link Expr.InputBindingTypes} to use to supply the type information of a
* Helper that can wrap another {@link Expr.InputBindingInspector} to use to supply the type information of a
* {@link LambdaExpr} when evaluating {@link ApplyFunctionExpr#getOutputType}. Lambda identifiers do not exist
* in the underlying {@link Expr.InputBindingTypes}, but can be created by mapping the lambda identifiers to the
* in the underlying {@link Expr.InputBindingInspector}, but can be created by mapping the lambda identifiers to the
* arguments that will be applied to them, to map the type information.
*/
class LambdaInputBindingTypes implements Expr.InputBindingTypes
class LambdaInputBindingInspector implements Expr.InputBindingInspector
{
private final Object2IntMap<String> lambdaIdentifiers;
private final Expr.InputBindingTypes inputTypes;
private final Expr.InputBindingInspector inspector;
private final List<Expr> args;

public LambdaInputBindingTypes(Expr.InputBindingTypes inputTypes, LambdaExpr expr, List<Expr> args)
public LambdaInputBindingInspector(Expr.InputBindingInspector inspector, LambdaExpr expr, List<Expr> args)
{
this.inputTypes = inputTypes;
this.inspector = inspector;
this.args = args;
List<String> identifiers = expr.getIdentifiers();
this.lambdaIdentifiers = new Object2IntOpenHashMap<>(args.size());
Expand All @@ -944,9 +948,9 @@ public LambdaInputBindingTypes(Expr.InputBindingTypes inputTypes, LambdaExpr exp
public ExprType getType(String name)
{
if (lambdaIdentifiers.containsKey(name)) {
return ExprType.elementType(args.get(lambdaIdentifiers.getInt(name)).getOutputType(inputTypes));
return ExprType.elementType(args.get(lambdaIdentifiers.getInt(name)).getOutputType(inspector));
}
return inputTypes.getType(name);
return inspector.getType(name);
}
}
}
72 changes: 36 additions & 36 deletions core/src/main/java/org/apache/druid/math/expr/BinaryLogicalOperatorExpr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,25 +62,25 @@ protected final double evalDouble(double left, double right)

@Nullable
@Override
public ExprType getOutputType(InputBindingTypes inputTypes)
public ExprType getOutputType(InputBindingInspector inspector)
{
ExprType implicitCast = super.getOutputType(inputTypes);
ExprType implicitCast = super.getOutputType(inspector);
if (ExprType.STRING.equals(implicitCast)) {
return ExprType.LONG;
}
return implicitCast;
}

@Override
public boolean canVectorize(InputBindingTypes inputTypes)
public boolean canVectorize(InputBindingInspector inspector)
{
return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right);
return inspector.areNumeric(left, right) && inspector.canVectorize(left, right);
}

@Override
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingTypes inputTypes)
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingInspector inspector)
{
return VectorComparisonProcessors.lessThan(inputTypes, left, right);
return VectorComparisonProcessors.lessThan(inspector, left, right);
}
}

Expand Down Expand Up @@ -118,25 +118,25 @@ protected final double evalDouble(double left, double right)

@Nullable
@Override
public ExprType getOutputType(InputBindingTypes inputTypes)
public ExprType getOutputType(InputBindingInspector inspector)
{
ExprType implicitCast = super.getOutputType(inputTypes);
ExprType implicitCast = super.getOutputType(inspector);
if (ExprType.STRING.equals(implicitCast)) {
return ExprType.LONG;
}
return implicitCast;
}

@Override
public boolean canVectorize(InputBindingTypes inputTypes)
public boolean canVectorize(InputBindingInspector inspector)
{
return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right);
return inspector.areNumeric(left, right) && inspector.canVectorize(left, right);
}

@Override
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingTypes inputTypes)
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingInspector inspector)
{
return VectorComparisonProcessors.lessThanOrEqual(inputTypes, left, right);
return VectorComparisonProcessors.lessThanOrEqual(inspector, left, right);
}
}

Expand Down Expand Up @@ -174,24 +174,24 @@ protected final double evalDouble(double left, double right)

@Nullable
@Override
public ExprType getOutputType(InputBindingTypes inputTypes)
public ExprType getOutputType(InputBindingInspector inspector)
{
ExprType implicitCast = super.getOutputType(inputTypes);
ExprType implicitCast = super.getOutputType(inspector);
if (ExprType.STRING.equals(implicitCast)) {
return ExprType.LONG;
}
return implicitCast;
}
@Override
public boolean canVectorize(InputBindingTypes inputTypes)
public boolean canVectorize(InputBindingInspector inspector)
{
return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right);
return inspector.areNumeric(left, right) && inspector.canVectorize(left, right);
}

@Override
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingTypes inputTypes)
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingInspector inspector)
{
return VectorComparisonProcessors.greaterThan(inputTypes, left, right);
return VectorComparisonProcessors.greaterThan(inspector, left, right);
}
}

Expand Down Expand Up @@ -229,25 +229,25 @@ protected final double evalDouble(double left, double right)

@Nullable
@Override
public ExprType getOutputType(InputBindingTypes inputTypes)
public ExprType getOutputType(InputBindingInspector inspector)
{
ExprType implicitCast = super.getOutputType(inputTypes);
ExprType implicitCast = super.getOutputType(inspector);
if (ExprType.STRING.equals(implicitCast)) {
return ExprType.LONG;
}
return implicitCast;
}

@Override
public boolean canVectorize(InputBindingTypes inputTypes)
public boolean canVectorize(InputBindingInspector inspector)
{
return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right);
return inspector.areNumeric(left, right) && inspector.canVectorize(left, right);
}

@Override
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingTypes inputTypes)
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingInspector inspector)
{
return VectorComparisonProcessors.greaterThanOrEqual(inputTypes, left, right);
return VectorComparisonProcessors.greaterThanOrEqual(inspector, left, right);
}
}

Expand Down Expand Up @@ -284,25 +284,25 @@ protected final double evalDouble(double left, double right)

@Nullable
@Override
public ExprType getOutputType(InputBindingTypes inputTypes)
public ExprType getOutputType(InputBindingInspector inspector)
{
ExprType implicitCast = super.getOutputType(inputTypes);
ExprType implicitCast = super.getOutputType(inspector);
if (ExprType.STRING.equals(implicitCast)) {
return ExprType.LONG;
}
return implicitCast;
}

@Override
public boolean canVectorize(InputBindingTypes inputTypes)
public boolean canVectorize(InputBindingInspector inspector)
{
return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right);
return inspector.areNumeric(left, right) && inspector.canVectorize(left, right);
}

@Override
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingTypes inputTypes)
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingInspector inspector)
{
return VectorComparisonProcessors.equal(inputTypes, left, right);
return VectorComparisonProcessors.equal(inspector, left, right);
}
}

Expand Down Expand Up @@ -339,25 +339,25 @@ protected final double evalDouble(double left, double right)

@Nullable
@Override
public ExprType getOutputType(InputBindingTypes inputTypes)
public ExprType getOutputType(InputBindingInspector inspector)
{
ExprType implicitCast = super.getOutputType(inputTypes);
ExprType implicitCast = super.getOutputType(inspector);
if (ExprType.STRING.equals(implicitCast)) {
return ExprType.LONG;
}
return implicitCast;
}

@Override
public boolean canVectorize(InputBindingTypes inputTypes)
public boolean canVectorize(InputBindingInspector inspector)
{
return inputTypes.areNumeric(left, right) && inputTypes.canVectorize(left, right);
return inspector.areNumeric(left, right) && inspector.canVectorize(left, right);
}

@Override
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingTypes inputTypes)
public <T> ExprVectorProcessor<T> buildVectorized(VectorInputBindingInspector inspector)
{
return VectorComparisonProcessors.notEqual(inputTypes, left, right);
return VectorComparisonProcessors.notEqual(inspector, left, right);
}
}

Expand Down
Loading