Skip to content

[bugfix](iceberg)Convert the datetime type in the predicate according to the target column for 2.0 #33092

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaokang merged 1 commit into from
Apr 2, 2024
Merged

[bugfix](iceberg)Convert the datetime type in the predicate according to the target column for 2.0 #33092

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 155 additions & 60 deletions fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@
import org.apache.iceberg.TableScan;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Unbound;
import org.apache.iceberg.transforms.PartitionSpecVisitor;
import org.apache.iceberg.types.Type.TypeID;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -234,51 +236,46 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) {
return null;
}

Expression expression = null;
// BoolLiteral
if (expr instanceof BoolLiteral) {
BoolLiteral boolLiteral = (BoolLiteral) expr;
boolean value = boolLiteral.getValue();
if (value) {
return Expressions.alwaysTrue();
expression = Expressions.alwaysTrue();
} else {
return Expressions.alwaysFalse();
expression = Expressions.alwaysFalse();
}
}

// CompoundPredicate
if (expr instanceof CompoundPredicate) {
} else if (expr instanceof CompoundPredicate) {
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
switch (compoundPredicate.getOp()) {
case AND: {
Expression left = convertToIcebergExpr(compoundPredicate.getChild(0), schema);
Expression right = convertToIcebergExpr(compoundPredicate.getChild(1), schema);
if (left != null && right != null) {
return Expressions.and(left, right);
expression = Expressions.and(left, right);
}
return null;
break;
}
case OR: {
Expression left = convertToIcebergExpr(compoundPredicate.getChild(0), schema);
Expression right = convertToIcebergExpr(compoundPredicate.getChild(1), schema);
if (left != null && right != null) {
return Expressions.or(left, right);
expression = Expressions.or(left, right);
}
return null;
break;
}
case NOT: {
Expression child = convertToIcebergExpr(compoundPredicate.getChild(0), schema);
if (child != null) {
return Expressions.not(child);
expression = Expressions.not(child);
}
return null;
break;
}
default:
return null;
}
}

// BinaryPredicate
if (expr instanceof BinaryPredicate) {
} else if (expr instanceof BinaryPredicate) {
TExprOpcode opCode = expr.getOpcode();
switch (opCode) {
case EQ:
Expand All @@ -303,38 +300,44 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) {
String colName = slotRef.getColumnName();
Types.NestedField nestedField = schema.caseInsensitiveFindField(colName);
colName = nestedField.name();
Object value = extractDorisLiteral(literalExpr);
Object value = extractDorisLiteral(nestedField.type(), literalExpr);
if (value == null) {
if (opCode == TExprOpcode.EQ_FOR_NULL && literalExpr instanceof NullLiteral) {
return Expressions.isNull(colName);
expression = Expressions.isNull(colName);
} else {
return null;
}
} else {
switch (opCode) {
case EQ:
case EQ_FOR_NULL:
expression = Expressions.equal(colName, value);
break;
case NE:
expression = Expressions.not(Expressions.equal(colName, value));
break;
case GE:
expression = Expressions.greaterThanOrEqual(colName, value);
break;
case GT:
expression = Expressions.greaterThan(colName, value);
break;
case LE:
expression = Expressions.lessThanOrEqual(colName, value);
break;
case LT:
expression = Expressions.lessThan(colName, value);
break;
default:
return null;
}
}
switch (opCode) {
case EQ:
case EQ_FOR_NULL:
return Expressions.equal(colName, value);
case NE:
return Expressions.not(Expressions.equal(colName, value));
case GE:
return Expressions.greaterThanOrEqual(colName, value);
case GT:
return Expressions.greaterThan(colName, value);
case LE:
return Expressions.lessThanOrEqual(colName, value);
case LT:
return Expressions.lessThan(colName, value);
default:
return null;
}
break;
default:
return null;
}
}

// InPredicate, only support a in (1,2,3)
if (expr instanceof InPredicate) {
} else if (expr instanceof InPredicate) {
// InPredicate, only support a in (1,2,3)
InPredicate inExpr = (InPredicate) expr;
if (inExpr.contains(Subquery.class)) {
return null;
Expand All @@ -343,56 +346,148 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) {
if (slotRef == null) {
return null;
}
String colName = slotRef.getColumnName();
Types.NestedField nestedField = schema.caseInsensitiveFindField(colName);
colName = nestedField.name();
List<Object> valueList = new ArrayList<>();
for (int i = 1; i < inExpr.getChildren().size(); ++i) {
if (!(inExpr.getChild(i) instanceof LiteralExpr)) {
return null;
}
LiteralExpr literalExpr = (LiteralExpr) inExpr.getChild(i);
Object value = extractDorisLiteral(literalExpr);
Object value = extractDorisLiteral(nestedField.type(), literalExpr);
valueList.add(value);
}
String colName = slotRef.getColumnName();
Types.NestedField nestedField = schema.caseInsensitiveFindField(colName);
colName = nestedField.name();
if (inExpr.isNotIn()) {
// not in
return Expressions.notIn(colName, valueList);
expression = Expressions.notIn(colName, valueList);
} else {
// in
return Expressions.in(colName, valueList);
expression = Expressions.in(colName, valueList);
}
}
if (expression != null && expression instanceof Unbound) {
try {
((Unbound<?, ?>) expression).bind(schema.asStruct(), true);
return expression;
} catch (Exception e) {
LOG.warn("Failed to check expression: " + e.getMessage());
return null;
}
}

return null;
}

private static Object extractDorisLiteral(Expr expr) {
if (!expr.isLiteral()) {
return null;
}
private static Object extractDorisLiteral(org.apache.iceberg.types.Type icebergType, Expr expr) {
TypeID icebergTypeID = icebergType.typeId();
if (expr instanceof BoolLiteral) {
BoolLiteral boolLiteral = (BoolLiteral) expr;
return boolLiteral.getValue();
switch (icebergTypeID) {
case BOOLEAN:
return boolLiteral.getValue();
case STRING:
return boolLiteral.getStringValue();
default:
return null;
}
} else if (expr instanceof DateLiteral) {
DateLiteral dateLiteral = (DateLiteral) expr;
if (dateLiteral.isDateType() || dateLiteral.isDateTimeType()) {
return dateLiteral.getStringValue();
} else {
return dateLiteral.unixTimestamp(TimeUtils.getTimeZone()) * MILLIS_TO_NANO_TIME;
switch (icebergTypeID) {
case STRING:
return dateLiteral.getStringValue();
case TIMESTAMP:
return dateLiteral.unixTimestamp(TimeUtils.getTimeZone()) * MILLIS_TO_NANO_TIME;
default:
return null;
}
} else if (expr instanceof DecimalLiteral) {
DecimalLiteral decimalLiteral = (DecimalLiteral) expr;
return decimalLiteral.getValue();
switch (icebergTypeID) {
case DECIMAL:
return decimalLiteral.getValue();
case STRING:
return decimalLiteral.getStringValue();
case DOUBLE:
return decimalLiteral.getDoubleValue();
default:
return null;
}
} else if (expr instanceof FloatLiteral) {
FloatLiteral floatLiteral = (FloatLiteral) expr;
return floatLiteral.getValue();
if (floatLiteral.getType() == Type.FLOAT) {
switch (icebergTypeID) {
case FLOAT:
case DOUBLE:
case DECIMAL:
return floatLiteral.getValue();
default:
return null;
}
} else {
switch (icebergTypeID) {
case DOUBLE:
case DECIMAL:
return floatLiteral.getValue();
default:
return null;
}
}
} else if (expr instanceof IntLiteral) {
IntLiteral intLiteral = (IntLiteral) expr;
return intLiteral.getValue();
Type type = intLiteral.getType();
if (type.isInteger32Type()) {
switch (icebergTypeID) {
case INTEGER:
case LONG:
case FLOAT:
case DOUBLE:
case DATE:
case DECIMAL:
return (int) intLiteral.getValue();
default:
return null;
}
} else {
// only PrimitiveType.BIGINT
switch (icebergTypeID) {
case INTEGER:
case LONG:
case FLOAT:
case DOUBLE:
case TIME:
case TIMESTAMP:
case DATE:
case DECIMAL:
return intLiteral.getValue();
default:
return null;
}
}
} else if (expr instanceof StringLiteral) {
StringLiteral stringLiteral = (StringLiteral) expr;
return stringLiteral.getStringValue();
String value = expr.getStringValue();
switch (icebergTypeID) {
case DATE:
case TIME:
case TIMESTAMP:
case STRING:
case UUID:
case DECIMAL:
return value;
case INTEGER:
try {
return Integer.parseInt(value);
} catch (Exception e) {
return null;
}
case LONG:
try {
return Long.parseLong(value);
} catch (Exception e) {
return null;
}
default:
return null;
}
}
return null;
}
Expand Down
Loading