Skip to content

[fix](nereids)should use nereids expr's nullable info when call Expr's toThrift method #35071

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
starocean999 merged 1 commit into from
May 23, 2024
Merged

[fix](nereids)should use nereids expr's nullable info when call Expr's toThrift method #35071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl

public static final float FUNCTION_CALL_COST = 10;

protected Optional<Boolean> nullableFromNereids = Optional.empty();

// returns true if an Expr is a non-analytic aggregate.
private static final com.google.common.base.Predicate<Expr> IS_AGGREGATE_PREDICATE =
new com.google.common.base.Predicate<Expr>() {
Expand Down Expand Up @@ -1003,7 +1005,7 @@ protected void treeToThriftHelper(TExpr container) {
}
}
msg.output_scale = getOutputScale();
msg.setIsNullable(isNullable());
msg.setIsNullable(nullableFromNereids.isPresent() ? nullableFromNereids.get() : isNullable());
toThrift(msg);
container.addToNodes(msg);
for (Expr child : children) {
Expand Down Expand Up @@ -2576,5 +2578,9 @@ public boolean isNullLiteral() {
public boolean isZeroLiteral() {
return this instanceof LiteralExpr && ((LiteralExpr) this).isZero();
}

public void setNullableFromNereids(boolean nullable) {
nullableFromNereids = Optional.of(nullable);
}
}

137 changes: 99 additions & 38 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,47 +141,57 @@ public Expr visitAlias(Alias alias, PlanTranslatorContext context) {

@Override
public Expr visitEqualTo(EqualTo equalTo, PlanTranslatorContext context) {
return new BinaryPredicate(Operator.EQ,
BinaryPredicate eq = new BinaryPredicate(Operator.EQ,
equalTo.child(0).accept(this, context),
equalTo.child(1).accept(this, context),
equalTo.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT);
eq.setNullableFromNereids(equalTo.nullable());
return eq;
}

@Override
public Expr visitGreaterThan(GreaterThan greaterThan, PlanTranslatorContext context) {
return new BinaryPredicate(Operator.GT,
BinaryPredicate gt = new BinaryPredicate(Operator.GT,
greaterThan.child(0).accept(this, context),
greaterThan.child(1).accept(this, context),
greaterThan.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT);
gt.setNullableFromNereids(greaterThan.nullable());
return gt;
}

@Override
public Expr visitGreaterThanEqual(GreaterThanEqual greaterThanEqual, PlanTranslatorContext context) {
return new BinaryPredicate(Operator.GE,
BinaryPredicate ge = new BinaryPredicate(Operator.GE,
greaterThanEqual.child(0).accept(this, context),
greaterThanEqual.child(1).accept(this, context),
greaterThanEqual.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT);
ge.setNullableFromNereids(greaterThanEqual.nullable());
return ge;
}

@Override
public Expr visitLessThan(LessThan lessThan, PlanTranslatorContext context) {
return new BinaryPredicate(Operator.LT,
BinaryPredicate lt = new BinaryPredicate(Operator.LT,
lessThan.child(0).accept(this, context),
lessThan.child(1).accept(this, context),
lessThan.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT);
lt.setNullableFromNereids(lessThan.nullable());
return lt;
}

@Override
public Expr visitLessThanEqual(LessThanEqual lessThanEqual, PlanTranslatorContext context) {
return new BinaryPredicate(Operator.LE,
BinaryPredicate le = new BinaryPredicate(Operator.LE,
lessThanEqual.child(0).accept(this, context),
lessThanEqual.child(1).accept(this, context),
lessThanEqual.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT);
le.setNullableFromNereids(lessThanEqual.nullable());
return le;
}

private OlapTable getOlapTableFromSlotDesc(SlotDescriptor slotDesc) {
Expand Down Expand Up @@ -248,23 +258,23 @@ public Expr visitMatch(Match match, PlanTranslatorContext context) {
}

MatchPredicate.Operator op = match.op();
return new MatchPredicate(op,
match.left().accept(this, context),
match.right().accept(this, context),
match.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT,
invertedIndexParser,
invertedIndexParserMode,
invertedIndexCharFilter);
MatchPredicate matchPredicate = new MatchPredicate(op, match.left().accept(this, context),
match.right().accept(this, context), match.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT, invertedIndexParser, invertedIndexParserMode,
invertedIndexCharFilter);
matchPredicate.setNullableFromNereids(match.nullable());
return matchPredicate;
}

@Override
public Expr visitNullSafeEqual(NullSafeEqual nullSafeEqual, PlanTranslatorContext context) {
return new BinaryPredicate(Operator.EQ_FOR_NULL,
BinaryPredicate eq = new BinaryPredicate(Operator.EQ_FOR_NULL,
nullSafeEqual.child(0).accept(this, context),
nullSafeEqual.child(1).accept(this, context),
nullSafeEqual.getDataType().toCatalogDataType(),
NullableMode.ALWAYS_NOT_NULLABLE);
eq.setNullableFromNereids(nullSafeEqual.nullable());
return eq;
}

@Override
Expand All @@ -275,23 +285,32 @@ public Expr visitNot(Not not, PlanTranslatorContext context) {
.map(e -> translate(e, context))
.collect(Collectors.toList());
boolean allConstant = inPredicate.getOptions().stream().allMatch(Expression::isConstant);
return new org.apache.doris.analysis.InPredicate(
org.apache.doris.analysis.InPredicate in = new org.apache.doris.analysis.InPredicate(
inPredicate.getCompareExpr().accept(this, context),
inList, true, allConstant);
in.setNullableFromNereids(inPredicate.nullable());
return in;
} else if (not.child() instanceof EqualTo) {
EqualTo equalTo = (EqualTo) not.child();
return new BinaryPredicate(Operator.NE,
BinaryPredicate ne = new BinaryPredicate(Operator.NE,
equalTo.child(0).accept(this, context),
equalTo.child(1).accept(this, context),
equalTo.getDataType().toCatalogDataType(),
NullableMode.DEPEND_ON_ARGUMENT);
ne.setNullableFromNereids(equalTo.nullable());
return ne;
} else if (not.child() instanceof InSubquery || not.child() instanceof Exists) {
return new BoolLiteral(true);
} else if (not.child() instanceof IsNull) {
return new IsNullPredicate(((IsNull) not.child()).child().accept(this, context), true, true);
IsNullPredicate isNull = new IsNullPredicate(
((IsNull) not.child()).child().accept(this, context), true, true);
isNull.setNullableFromNereids(not.child().nullable());
return isNull;
} else {
return new CompoundPredicate(CompoundPredicate.Operator.NOT,
CompoundPredicate cp = new CompoundPredicate(CompoundPredicate.Operator.NOT,
not.child(0).accept(this, context), null);
cp.setNullableFromNereids(not.nullable());
return cp;
}
}

Expand Down Expand Up @@ -330,18 +349,22 @@ public Expr visitNullLiteral(NullLiteral nullLiteral, PlanTranslatorContext cont

@Override
public Expr visitAnd(And and, PlanTranslatorContext context) {
return new org.apache.doris.analysis.CompoundPredicate(
org.apache.doris.analysis.CompoundPredicate cp = new org.apache.doris.analysis.CompoundPredicate(
org.apache.doris.analysis.CompoundPredicate.Operator.AND,
and.child(0).accept(this, context),
and.child(1).accept(this, context));
cp.setNullableFromNereids(and.nullable());
return cp;
}

@Override
public Expr visitOr(Or or, PlanTranslatorContext context) {
return new org.apache.doris.analysis.CompoundPredicate(
org.apache.doris.analysis.CompoundPredicate cp = new org.apache.doris.analysis.CompoundPredicate(
org.apache.doris.analysis.CompoundPredicate.Operator.OR,
or.child(0).accept(this, context),
or.child(1).accept(this, context));
cp.setNullableFromNereids(or.nullable());
return cp;
}

@Override
Expand All @@ -358,14 +381,18 @@ public Expr visitCaseWhen(CaseWhen caseWhen, PlanTranslatorContext context) {
if (defaultValue.isPresent()) {
elseExpr = defaultValue.get().accept(this, context);
}
return new CaseExpr(caseWhenClauses, elseExpr);
CaseExpr caseExpr = new CaseExpr(caseWhenClauses, elseExpr);
caseExpr.setNullableFromNereids(caseWhen.nullable());
return caseExpr;
}

@Override
public Expr visitCast(Cast cast, PlanTranslatorContext context) {
// left child of cast is expression, right child of cast is target type
return new CastExpr(cast.getDataType().toCatalogDataType(),
CastExpr castExpr = new CastExpr(cast.getDataType().toCatalogDataType(),
cast.child().accept(this, context), null);
castExpr.setNullableFromNereids(cast.nullable());
return castExpr;
}

@Override
Expand All @@ -374,9 +401,11 @@ public Expr visitInPredicate(InPredicate inPredicate, PlanTranslatorContext cont
.map(e -> e.accept(this, context))
.collect(Collectors.toList());
boolean allConstant = inPredicate.getOptions().stream().allMatch(Expression::isConstant);
return new org.apache.doris.analysis.InPredicate(
org.apache.doris.analysis.InPredicate in = new org.apache.doris.analysis.InPredicate(
inPredicate.getCompareExpr().accept(this, context),
inList, false, allConstant);
in.setNullableFromNereids(inPredicate.nullable());
return in;
}

@Override
Expand Down Expand Up @@ -420,6 +449,7 @@ public Expr visitWindowFunction(WindowFunction function, PlanTranslatorContext c
FunctionCallExpr functionCallExpr =
new FunctionCallExpr(catalogFunction, windowFnParams, windowFnParams, isMergeFn, catalogArguments);
functionCallExpr.setIsAnalyticFnCall(true);
functionCallExpr.setNullableFromNereids(function.nullable());
return functionCallExpr;

}
Expand All @@ -429,7 +459,9 @@ public Expr visitLambda(Lambda lambda, PlanTranslatorContext context) {
Expr func = lambda.getLambdaFunction().accept(this, context);
List<Expr> arguments = lambda.getLambdaArguments().stream().map(e -> e.accept(this, context))
.collect(Collectors.toList());
return new LambdaFunctionExpr(func, lambda.getLambdaArgumentNames(), arguments);
LambdaFunctionExpr functionExpr = new LambdaFunctionExpr(func, lambda.getLambdaArgumentNames(), arguments);
functionExpr.setNullableFromNereids(lambda.nullable());
return functionExpr;
}

@Override
Expand Down Expand Up @@ -472,7 +504,10 @@ public Expr visitArrayMap(ArrayMap arrayMap, PlanTranslatorContext context) {
// create catalog FunctionCallExpr without analyze again
Expr lambdaBody = visitLambda(lambda, context);
arguments.set(0, lambdaBody);
return new LambdaFunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
LambdaFunctionCallExpr functionCallExpr =
new LambdaFunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
functionCallExpr.setNullableFromNereids(arrayMap.nullable());
return functionCallExpr;
}

@Override
Expand All @@ -495,11 +530,15 @@ public Expr visitScalarFunction(ScalarFunction function, PlanTranslatorContext c
function.getDataType().toCatalogDataType(), function.hasVarArguments(),
"", TFunctionBinaryType.BUILTIN, true, true, nullableMode);

FunctionCallExpr functionCallExpr;
// create catalog FunctionCallExpr without analyze again
if (function instanceof HighOrderFunction) {
return new LambdaFunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
functionCallExpr = new LambdaFunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
} else {
functionCallExpr = new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
}
return new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
functionCallExpr.setNullableFromNereids(function.nullable());
return functionCallExpr;
}

@Override
Expand Down Expand Up @@ -539,7 +578,10 @@ public Expr visitTableGeneratingFunction(TableGeneratingFunction function,
"", TFunctionBinaryType.BUILTIN, true, true, nullableMode);

// create catalog FunctionCallExpr without analyze again
return new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
FunctionCallExpr functionCallExpr =
new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
functionCallExpr.setNullableFromNereids(function.nullable());
return functionCallExpr;
}

@Override
Expand All @@ -550,18 +592,21 @@ public Expr visitBinaryArithmetic(BinaryArithmetic binaryArithmetic, PlanTransla
} else if (binaryArithmetic instanceof AlwaysNotNullable) {
nullableMode = NullableMode.ALWAYS_NOT_NULLABLE;
}
return new ArithmeticExpr(binaryArithmetic.getLegacyOperator(),
ArithmeticExpr arithmeticExpr = new ArithmeticExpr(binaryArithmetic.getLegacyOperator(),
binaryArithmetic.child(0).accept(this, context),
binaryArithmetic.child(1).accept(this, context),
binaryArithmetic.getDataType().toCatalogDataType(), nullableMode);
arithmeticExpr.setNullableFromNereids(binaryArithmetic.nullable());
return arithmeticExpr;
}

@Override
public Expr visitUnaryArithmetic(UnaryArithmetic unaryArithmetic, PlanTranslatorContext context) {
return new ArithmeticExpr(unaryArithmetic.getLegacyOperator(),
ArithmeticExpr arithmeticExpr = new ArithmeticExpr(unaryArithmetic.getLegacyOperator(),
unaryArithmetic.child().accept(this, context), null,
unaryArithmetic.getDataType().toCatalogDataType(), NullableMode.DEPEND_ON_ARGUMENT);

arithmeticExpr.setNullableFromNereids(unaryArithmetic.nullable());
return arithmeticExpr;
}

@Override
Expand All @@ -572,9 +617,13 @@ public Expr visitTimestampArithmetic(TimestampArithmetic arithmetic, PlanTransla
if (arithmetic.children().stream().anyMatch(e -> e.getDataType().isDateV2LikeType())) {
nullableMode = NullableMode.DEPEND_ON_ARGUMENT;
}
return new TimestampArithmeticExpr(arithmetic.getFuncName(), arithmetic.getOp(),
TimestampArithmeticExpr timestampArithmeticExpr = new TimestampArithmeticExpr(
arithmetic.getFuncName(), arithmetic.getOp(),
arithmetic.left().accept(this, context), arithmetic.right().accept(this, context),
arithmetic.getTimeUnit().toString(), arithmetic.getDataType().toCatalogDataType(), nullableMode);
arithmetic.getTimeUnit().toString(), arithmetic.getDataType().toCatalogDataType(),
nullableMode);
timestampArithmeticExpr.setNullableFromNereids(arithmetic.nullable());
return timestampArithmeticExpr;
}

@Override
Expand All @@ -584,7 +633,9 @@ public Expr visitVirtualReference(VirtualSlotReference virtualSlotReference, Pla

@Override
public Expr visitIsNull(IsNull isNull, PlanTranslatorContext context) {
return new IsNullPredicate(isNull.child().accept(this, context), false, true);
IsNullPredicate isNullPredicate = new IsNullPredicate(isNull.child().accept(this, context), false, true);
isNullPredicate.setNullableFromNereids(isNull.nullable());
return isNullPredicate;
}

@Override
Expand Down Expand Up @@ -640,31 +691,40 @@ public Expr visitAggregateFunction(AggregateFunction function, PlanTranslatorCon
TFunctionBinaryType.BUILTIN, true, true,
function.nullable() ? NullableMode.ALWAYS_NULLABLE : NullableMode.ALWAYS_NOT_NULLABLE);

return new FunctionCallExpr(catalogFunction, new FunctionParams(function.isDistinct(), arguments));
FunctionCallExpr functionCallExpr = new FunctionCallExpr(catalogFunction,
new FunctionParams(function.isDistinct(), arguments));
functionCallExpr.setNullableFromNereids(function.nullable());
return functionCallExpr;
}

@Override
public Expr visitJavaUdf(JavaUdf udf, PlanTranslatorContext context) {
FunctionParams exprs = new FunctionParams(udf.children().stream()
.map(expression -> expression.accept(this, context))
.collect(Collectors.toList()));
return new FunctionCallExpr(udf.getCatalogFunction(), exprs);
FunctionCallExpr functionCallExpr = new FunctionCallExpr(udf.getCatalogFunction(), exprs);
functionCallExpr.setNullableFromNereids(udf.nullable());
return functionCallExpr;
}

@Override
public Expr visitJavaUdtf(JavaUdtf udf, PlanTranslatorContext context) {
FunctionParams exprs = new FunctionParams(udf.children().stream()
.map(expression -> expression.accept(this, context))
.collect(Collectors.toList()));
return new FunctionCallExpr(udf.getCatalogFunction(), exprs);
FunctionCallExpr functionCallExpr = new FunctionCallExpr(udf.getCatalogFunction(), exprs);
functionCallExpr.setNullableFromNereids(udf.nullable());
return functionCallExpr;
}

@Override
public Expr visitJavaUdaf(JavaUdaf udaf, PlanTranslatorContext context) {
FunctionParams exprs = new FunctionParams(udaf.isDistinct(), udaf.children().stream()
.map(expression -> expression.accept(this, context))
.collect(Collectors.toList()));
return new FunctionCallExpr(udaf.getCatalogFunction(), exprs);
FunctionCallExpr functionCallExpr = new FunctionCallExpr(udaf.getCatalogFunction(), exprs);
functionCallExpr.setNullableFromNereids(udaf.nullable());
return functionCallExpr;
}

// TODO: Supports for `distinct`
Expand Down Expand Up @@ -707,6 +767,7 @@ private Expr translateAggregateFunction(AggregateFunction function,
FunctionCallExpr functionCallExpr = new FunctionCallExpr(
catalogFunction, fnParams, aggFnParams, isMergeFn, currentPhaseCatalogArguments);
functionCallExpr.setOrderByElements(orderByElements);
functionCallExpr.setNullableFromNereids(function.nullable());
return functionCallExpr;
}

Expand Down