From 19b78e582c93ff65122692fd3924046b62d71ece Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 6 Apr 2024 20:59:43 +0800 Subject: [PATCH 1/7] 1 --- .../datasource/iceberg/IcebergUtils.java | 1 + ...Predict.java => IcebergPredicateTest.java} | 26 +++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) rename fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/{TestIcebergPredict.java => IcebergPredicateTest.java} (84%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 76b5bfb5105c4f..d05a911d71e582 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -248,6 +248,7 @@ public static Object extractDorisLiteral(org.apache.iceberg.types.Type icebergTy DateLiteral dateLiteral = (DateLiteral) expr; switch (icebergTypeID) { case STRING: + case DATE: return dateLiteral.getStringValue(); case TIMESTAMP: return dateLiteral.unixTimestamp(TimeUtils.getTimeZone()) * MILLIS_TO_NANO_TIME; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java similarity index 84% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java index 80b1c62819bb97..59c76ecf5666b3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/TestIcebergPredict.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java @@ -42,22 +42,22 @@ import java.util.List; import java.util.stream.Collectors; -public class TestIcebergPredict { +public class IcebergPredicateTest { public static Schema schema; @BeforeClass public static void before() throws AnalysisException { schema = new Schema( - Types.NestedField.required(1, "c_int", Types.IntegerType.get()), - Types.NestedField.required(2, "c_long", Types.LongType.get()), - Types.NestedField.required(3, "c_bool", Types.BooleanType.get()), - Types.NestedField.required(4, "c_float", Types.FloatType.get()), - Types.NestedField.required(5, "c_double", Types.DoubleType.get()), - Types.NestedField.required(6, "c_dec", Types.DecimalType.of(20, 10)), - Types.NestedField.required(7, "c_date", Types.DateType.get()), - Types.NestedField.required(8, "c_ts", Types.TimestampType.withoutZone()), - Types.NestedField.required(10, "c_str", Types.StringType.get()) + Types.NestedField.required(1, "c_int", Types.IntegerType.get()), + Types.NestedField.required(2, "c_long", Types.LongType.get()), + Types.NestedField.required(3, "c_bool", Types.BooleanType.get()), + Types.NestedField.required(4, "c_float", Types.FloatType.get()), + Types.NestedField.required(5, "c_double", Types.DoubleType.get()), + Types.NestedField.required(6, "c_dec", Types.DecimalType.of(20, 10)), + Types.NestedField.required(7, "c_date", Types.DateType.get()), + Types.NestedField.required(8, "c_ts", Types.TimestampType.withoutZone()), + Types.NestedField.required(10, "c_str", Types.StringType.get()) ); } @@ -111,9 +111,9 @@ public void testBinaryPredicate() throws AnalysisException { { // decimal false, false, false, true, true, true, true, true, true, true, false, false, false }, - { // date - false, false, false, false, false, false, true, true, true, true, false, true, false - }, + { // date + false, true, false, false, false, false, true, true, true, true, false, true, false + }, { // timestamp false, true, true, false, false, false, false, false, false, true, false, false, false }, From 4877fc9cb8544550b0d285d86c122fd92e9c32bf Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 6 Apr 2024 22:31:17 +0800 Subject: [PATCH 2/7] 1 --- .../iceberg/source/IcebergScanNode.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index e2564eae527015..65aeedb71dbe36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -40,6 +40,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileType; @@ -50,6 +51,7 @@ import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import org.apache.hadoop.fs.Path; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CombinedScanTask; @@ -90,6 +92,7 @@ public class IcebergScanNode extends FileQueryScanNode { private IcebergSource source; private Table icebergTable; + private List pushdownIcebergPredicates = Lists.newArrayList(); /** * External file scan node for Query iceberg table @@ -180,7 +183,6 @@ public List getSplits() throws UserException { } private List doGetSplits() throws UserException { - TableScan scan = icebergTable.newScan(); // set snapshot @@ -199,6 +201,7 @@ private List doGetSplits() throws UserException { } for (Expression predicate : expressions) { scan = scan.filter(predicate); + this.pushdownIcebergPredicates.add(predicate.toString()); } // get splits @@ -439,4 +442,14 @@ protected void toThrift(TPlanNode planNode) { } } } + + @Override + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + StringBuilder sb = new StringBuilder(); + for (String predicate : pushdownIcebergPredicates) { + sb.append(prefix).append(prefix).append(predicate).append("\n"); + } + return super.getNodeExplainString(prefix, detailLevel) + + String.format("%sicebergPredicatePushdown=\n%s\n", prefix, sb); + } } From b83c1acad839b5ebaac4ce903584fe65ced9f56d Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 6 Apr 2024 23:00:57 +0800 Subject: [PATCH 3/7] 1 --- .../datasource/iceberg/IcebergUtils.java | 65 ++++++++++++++++--- .../iceberg/IcebergPredicateTest.java | 48 +++++++------- 2 files changed, 81 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index d05a911d71e582..211497bfb7bf25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -48,8 +48,11 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.expressions.And; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Not; +import org.apache.iceberg.expressions.Or; import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; @@ -209,6 +212,9 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) { } LiteralExpr literalExpr = (LiteralExpr) inExpr.getChild(i); Object value = extractDorisLiteral(nestedField.type(), literalExpr); + if (value == null) { + return null; + } valueList.add(value); } if (inExpr.isNotIn()) { @@ -220,16 +226,59 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) { } } - if (expression != null && expression instanceof Unbound) { - try { - ((Unbound) expression).bind(schema.asStruct(), true); - return expression; - } catch (Exception e) { - LOG.warn("Failed to check expression: " + e.getMessage()); - return null; + return checkConversion(expression, schema); + } + + private static Expression checkConversion(Expression expression, Schema schema) { + if (expression == null) { + return null; + } + switch (expression.op()) { + case AND: { + And andExpr = (And) expression; + Expression left = checkConversion(andExpr.left(), schema); + Expression right = checkConversion(andExpr.right(), schema); + if (left != null && right != null) { + return andExpr; + } else if (left != null) { + return left; + } else if (right != null) { + return right; + } else { + return null; + } + } + case OR: { + Or orExpr = (Or) expression; + Expression left = checkConversion(orExpr.left(), schema); + Expression right = checkConversion(orExpr.right(), schema); + if (left == null || right == null) { + return null; + } else { + return orExpr; + } + } + case NOT: { + Not notExpr = (Not) expression; + Expression child = checkConversion(notExpr.child(), schema); + if (child == null) { + return null; + } else { + return notExpr; + } } + default: + if (!(expression instanceof Unbound)) { + return null; + } + try { + ((Unbound) expression).bind(schema.asStruct(), true); + return expression; + } catch (Exception e) { + LOG.warn("Failed to check expression: " + e.getMessage()); + return null; + } } - return null; } public static Object extractDorisLiteral(org.apache.iceberg.types.Type icebergType, Expr expr) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java index 59c76ecf5666b3..cd50262b12630c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java @@ -93,33 +93,33 @@ public void testBinaryPredicate() throws AnalysisException { // true indicates support for pushdown Boolean[][] expects = new Boolean[][] { - { // int - false, false, false, false, false, false, true, true, true, true, false, false, false - }, - { // long - false, false, false, false, false, false, true, true, true, true, false, false, false - }, - { // boolean - true, false, false, false, false, false, false, false, false, false, false, false, false - }, - { // float - false, false, false, false, true, false, true, true, true, true, false, false, false - }, - { // double - false, false, false, true, true, true, true, true, true, true, false, false, false - }, - { // decimal - false, false, false, true, true, true, true, true, true, true, false, false, false - }, + { // int + false, false, false, false, false, false, true, true, true, true, false, false, false + }, + { // long + false, false, false, false, false, false, true, true, true, true, false, false, false + }, + { // boolean + true, false, false, false, false, false, false, false, false, false, false, false, false + }, + { // float + false, false, false, false, true, false, true, true, true, true, false, false, false + }, + { // double + false, false, false, true, true, true, true, true, true, true, false, false, false + }, + { // decimal + false, false, false, true, true, true, true, true, true, true, false, false, false + }, { // date false, true, false, false, false, false, true, true, true, true, false, true, false }, - { // timestamp - false, true, true, false, false, false, false, false, false, true, false, false, false - }, - { // string - true, true, true, true, false, false, false, false, false, false, true, true, true - } + { // timestamp + false, true, true, false, false, false, false, false, false, true, false, false, false + }, + { // string + true, true, true, true, false, false, false, false, false, false, true, true, true + } }; for (int i = 0; i < slotRefs.size(); i++) { From 0e80d00395de80ed6c989d169085765479e4c46f Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 6 Apr 2024 23:38:17 +0800 Subject: [PATCH 4/7] 2 --- .../datasource/iceberg/IcebergUtils.java | 7 + .../iceberg/IcebergPredicateTest.java | 121 ++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 211497bfb7bf25..15128002e7cbb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -110,6 +110,10 @@ public static Expression convertToIcebergExpr(Expr expr, Schema schema) { Expression right = convertToIcebergExpr(compoundPredicate.getChild(1), schema); if (left != null && right != null) { expression = Expressions.and(left, right); + } else if (left != null) { + return left; + } else if (right != null) { + return right; } break; } @@ -267,6 +271,9 @@ private static Expression checkConversion(Expression expression, Schema schema) return notExpr; } } + case TRUE: + case FALSE: + return expression; default: if (!(expression instanceof Unbound)) { return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java index cd50262b12630c..f1471cb1ec47dd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPredicateTest.java @@ -19,9 +19,13 @@ import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.BoolLiteral; +import org.apache.doris.analysis.CompoundPredicate; +import org.apache.doris.analysis.CompoundPredicate.Operator; import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.DecimalLiteral; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FloatLiteral; +import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.SlotRef; @@ -30,8 +34,11 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.BeforeClass; @@ -122,14 +129,128 @@ public void testBinaryPredicate() throws AnalysisException { } }; + ArrayListMultimap validPredicateMap = ArrayListMultimap.create(); + + // binary predicate for (int i = 0; i < slotRefs.size(); i++) { final int loc = i; List ret = literalList.stream().map(literal -> { BinaryPredicate expr = new BinaryPredicate(BinaryPredicate.Operator.EQ, slotRefs.get(loc), literal); Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); + validPredicateMap.put(expression != null, expr); + return expression != null; + }).collect(Collectors.toList()); + Assert.assertArrayEquals(expects[i], ret.toArray()); + } + + // in predicate + for (int i = 0; i < slotRefs.size(); i++) { + final int loc = i; + List ret = literalList.stream().map(literal -> { + InPredicate expr = new InPredicate(slotRefs.get(loc), Lists.newArrayList(literal), false); + Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); + validPredicateMap.put(expression != null, expr); return expression != null; }).collect(Collectors.toList()); Assert.assertArrayEquals(expects[i], ret.toArray()); } + + // not in predicate + for (int i = 0; i < slotRefs.size(); i++) { + final int loc = i; + List ret = literalList.stream().map(literal -> { + InPredicate expr = new InPredicate(slotRefs.get(loc), Lists.newArrayList(literal), true); + Expression expression = IcebergUtils.convertToIcebergExpr(expr, schema); + validPredicateMap.put(expression != null, expr); + return expression != null; + }).collect(Collectors.toList()); + Assert.assertArrayEquals(expects[i], ret.toArray()); + } + + // bool literal + Expression trueExpr = IcebergUtils.convertToIcebergExpr(new BoolLiteral(true), schema); + Expression falseExpr = IcebergUtils.convertToIcebergExpr(new BoolLiteral(false), schema); + Assert.assertEquals(Expressions.alwaysTrue(), trueExpr); + Assert.assertEquals(Expressions.alwaysFalse(), falseExpr); + validPredicateMap.put(true, new BoolLiteral(true)); + validPredicateMap.put(true, new BoolLiteral(false)); + + List validExprs = validPredicateMap.get(true); + List invalidExprs = validPredicateMap.get(false); + // OR predicate + // both valid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < validExprs.size(); j++) { + CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR, + validExprs.get(i), validExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(orPredicate, schema); + Assert.assertNotNull("pred: " + orPredicate.toSql(), expression); + } + } + // both invalid + for (int i = 0; i < invalidExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR, + invalidExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(orPredicate, schema); + Assert.assertNull("pred: " + orPredicate.toSql(), expression); + } + } + // valid or invalid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate orPredicate = new CompoundPredicate(Operator.OR, + validExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(orPredicate, schema); + Assert.assertNull("pred: " + orPredicate.toSql(), expression); + } + } + + // AND predicate + // both valid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < validExprs.size(); j++) { + CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND, + validExprs.get(i), validExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(andPredicate, schema); + Assert.assertNotNull("pred: " + andPredicate.toSql(), expression); + } + } + // both invalid + for (int i = 0; i < invalidExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND, + invalidExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(andPredicate, schema); + Assert.assertNull("pred: " + andPredicate.toSql(), expression); + } + } + // valid and invalid + for (int i = 0; i < validExprs.size(); i++) { + for (int j = 0; j < invalidExprs.size(); j++) { + CompoundPredicate andPredicate = new CompoundPredicate(Operator.AND, + validExprs.get(i), invalidExprs.get(j)); + Expression expression = IcebergUtils.convertToIcebergExpr(andPredicate, schema); + Assert.assertNotNull("pred: " + andPredicate.toSql(), expression); + Assert.assertEquals(IcebergUtils.convertToIcebergExpr(validExprs.get(i), schema).toString(), + expression.toString()); + } + } + + // NOT predicate + // valid + for (int i = 0; i < validExprs.size(); i++) { + CompoundPredicate notPredicate = new CompoundPredicate(Operator.NOT, + validExprs.get(i), null); + Expression expression = IcebergUtils.convertToIcebergExpr(notPredicate, schema); + Assert.assertNotNull("pred: " + notPredicate.toSql(), expression); + } + // invalid + for (int i = 0; i < invalidExprs.size(); i++) { + CompoundPredicate notPredicate = new CompoundPredicate(Operator.NOT, + invalidExprs.get(i), null); + Expression expression = IcebergUtils.convertToIcebergExpr(notPredicate, schema); + Assert.assertNull("pred: " + notPredicate.toSql(), expression); + } } } From 56fdd54734e96c353cef6678552f9b38692298ba Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 6 Apr 2024 23:48:31 +0800 Subject: [PATCH 5/7] 3 --- .../java/org/apache/doris/datasource/iceberg/IcebergUtils.java | 2 +- .../doris/datasource/iceberg/source/IcebergScanNode.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 15128002e7cbb8..08c4be4ceace20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -282,7 +282,7 @@ private static Expression checkConversion(Expression expression, Schema schema) ((Unbound) expression).bind(schema.asStruct(), true); return expression; } catch (Exception e) { - LOG.warn("Failed to check expression: " + e.getMessage()); + LOG.debug("Failed to check expression: {}", e.getMessage()); return null; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 65aeedb71dbe36..961fb8ae1d6679 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -445,6 +445,9 @@ protected void toThrift(TPlanNode planNode) { @Override public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (pushdownIcebergPredicates.isEmpty()) { + return super.getNodeExplainString(prefix, detailLevel); + } StringBuilder sb = new StringBuilder(); for (String predicate : pushdownIcebergPredicates) { sb.append(prefix).append(prefix).append(predicate).append("\n"); From bad6c0adae69f32bd3ac8cee8669799cb1b8f296 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 6 Apr 2024 23:49:07 +0800 Subject: [PATCH 6/7] test --- .../test_iceberg_predicate_conversion.out | 611 ++++++++++++++++++ .../test_iceberg_predicate_conversion.groovy | 75 +++ 2 files changed, 686 insertions(+) create mode 100644 regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out create mode 100644 regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy diff --git a/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out b/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out new file mode 100644 index 00000000000000..0569e0d01d8375 --- /dev/null +++ b/regression-test/data/external_table_p2/iceberg/test_iceberg_predicate_conversion.out @@ -0,0 +1,611 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +11801003 35210325 + +-- !q02 -- +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1996-05-06 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 +1997-05-18 + +-- !q03 -- +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1996-05-06 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL +1997-05-18 MAIL + +-- !q04 -- +1992-01-02 REG AIR +1992-01-02 SHIP +1992-01-03 REG AIR +1992-01-03 TRUCK +1992-01-04 AIR +1992-01-04 FOB +1992-01-04 RAIL +1992-01-04 REG AIR +1992-01-04 TRUCK +1992-01-05 AIR + +-- !q04 -- +2023-03-07T20:35:59.064 +2023-03-07T20:35:59.087 +2023-03-07T20:35:59.110 +2023-03-07T20:35:59.129 +2023-03-07T20:35:59.224 + diff --git a/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy new file mode 100644 index 00000000000000..f6dc03639ec84d --- /dev/null +++ b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_predicate_conversion", "p2,external,hive,external_remote,external_remote_hive") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + + sql """drop catalog if exists test_iceberg_predicate_conversion;""" + sql """ + create catalog if not exists test_iceberg_predicate_conversion properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + + sql """switch test_iceberg_predicate_conversion;""" + sql """ use `iceberg_catalog`; """ + + def sqlstr = """select glue_int, glue_varchar from iceberg_glue_types where glue_varchar > date '2023-03-07' """ + def plan = """ref(name="glue_varchar") > "2023-03-07 00:00:00"""" + order_qt_q01 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """${plan}""" + } + + sqlstr = """select l_shipdate from lineitem where l_shipdate in ("1997-05-18", "1996-05-06"); """ + plan = """ref(name="l_shipdate") in ("1996-05-06", "1997-05-18")""" + order_qt_q02 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """${plan}""" + } + + sqlstr = """select l_shipdate, l_shipmode from lineitem where l_shipdate in ("1997-05-18", "1996-05-06") and l_shipmode = "MAIL";""" + order_qt_q03 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """ref(name="l_shipdate") in ("1996-05-06", "1997-05-18")""" + contains """ref(name="l_shipmode") == "MAIL"""" + } + + sqlstr = """select l_shipdate, l_shipmode from lineitem where l_shipdate in ("1997-05-18", "1996-05-06") or NOT(l_shipmode = "MAIL") order by l_shipdate, l_shipmode limit 10""" + plan = """(ref(name="l_shipdate") in ("1996-05-06", "1997-05-18") or not(ref(name="l_shipmode") == "MAIL"))""" + order_qt_q04 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """${plan}""" + } + + sqlstr = """select glue_timstamp from iceberg_glue_types where glue_timstamp > '2023-03-07 20:35:59' order by glue_timstamp limit 5""" + plan = """ref(name="glue_timstamp") > 1678192559000000""" + order_qt_q04 """${sqlstr}""" + explain { + sql("""${sqlstr}""") + contains """${plan}""" + } + } +} From 41354123a9523fc1b897d577616ae0a6cfd58a25 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 7 Apr 2024 08:29:42 +0800 Subject: [PATCH 7/7] fix test --- .../test_iceberg_predicate_conversion.groovy | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy index f6dc03639ec84d..58518489ef0e38 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_iceberg_predicate_conversion.groovy @@ -33,43 +33,47 @@ suite("test_iceberg_predicate_conversion", "p2,external,hive,external_remote,ext sql """ use `iceberg_catalog`; """ def sqlstr = """select glue_int, glue_varchar from iceberg_glue_types where glue_varchar > date '2023-03-07' """ - def plan = """ref(name="glue_varchar") > "2023-03-07 00:00:00"""" order_qt_q01 """${sqlstr}""" explain { sql("""${sqlstr}""") - contains """${plan}""" + contains """ref(name="glue_varchar") > "2023-03-07 00:00:00""" } sqlstr = """select l_shipdate from lineitem where l_shipdate in ("1997-05-18", "1996-05-06"); """ - plan = """ref(name="l_shipdate") in ("1996-05-06", "1997-05-18")""" order_qt_q02 """${sqlstr}""" explain { sql("""${sqlstr}""") - contains """${plan}""" + contains """ref(name="l_shipdate") in""" + contains """"1997-05-18"""" + contains """"1996-05-06"""" } sqlstr = """select l_shipdate, l_shipmode from lineitem where l_shipdate in ("1997-05-18", "1996-05-06") and l_shipmode = "MAIL";""" order_qt_q03 """${sqlstr}""" explain { sql("""${sqlstr}""") - contains """ref(name="l_shipdate") in ("1996-05-06", "1997-05-18")""" + contains """ref(name="l_shipdate") in""" + contains """"1997-05-18"""" + contains """"1996-05-06"""" contains """ref(name="l_shipmode") == "MAIL"""" } sqlstr = """select l_shipdate, l_shipmode from lineitem where l_shipdate in ("1997-05-18", "1996-05-06") or NOT(l_shipmode = "MAIL") order by l_shipdate, l_shipmode limit 10""" - plan = """(ref(name="l_shipdate") in ("1996-05-06", "1997-05-18") or not(ref(name="l_shipmode") == "MAIL"))""" + plan = """(ref(name="l_shipdate") in ("1997-05-18", "1996-05-06") or not(ref(name="l_shipmode") == "MAIL"))""" order_qt_q04 """${sqlstr}""" explain { sql("""${sqlstr}""") - contains """${plan}""" + contains """or not(ref(name="l_shipmode") == "MAIL"))""" + contains """ref(name="l_shipdate")""" + contains """"1997-05-18"""" + contains """"1996-05-06"""" } sqlstr = """select glue_timstamp from iceberg_glue_types where glue_timstamp > '2023-03-07 20:35:59' order by glue_timstamp limit 5""" - plan = """ref(name="glue_timstamp") > 1678192559000000""" order_qt_q04 """${sqlstr}""" explain { sql("""${sqlstr}""") - contains """${plan}""" + contains """ref(name="glue_timstamp") > 1678192559000000""" } } }