From c899b7bbdfd6c92c2b2d0b2cc8cedf5cc9a10038 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 6 Nov 2024 15:35:38 +0800 Subject: [PATCH 1/3] opt between --- .../nereids/parser/LogicalPlanBuilder.java | 14 +++++-- .../doris/nereids/parser/BetweenTest.java | 40 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/BetweenTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 8362d002d2b813..a038d0c85af776 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3504,10 +3504,16 @@ private Expression withPredicate(Expression valueExpression, PredicateContext ct Expression outExpression; switch (ctx.kind.getType()) { case DorisParser.BETWEEN: - outExpression = new And( - new GreaterThanEqual(valueExpression, getExpression(ctx.lower)), - new LessThanEqual(valueExpression, getExpression(ctx.upper)) - ); + Expression lower = getExpression(ctx.lower); + Expression upper = getExpression(ctx.upper); + if (lower.equals(upper)) { + outExpression = new EqualTo(valueExpression, lower); + } else { + outExpression = new And( + new GreaterThanEqual(valueExpression, getExpression(ctx.lower)), + new LessThanEqual(valueExpression, getExpression(ctx.upper)) + ); + } break; case DorisParser.LIKE: outExpression = new Like( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/BetweenTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/BetweenTest.java new file mode 100644 index 00000000000000..c9ef4fbee27cbe --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/BetweenTest.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class BetweenTest { + private static final NereidsParser PARSER = new NereidsParser(); + + @Test + public void testBetween() { + String expression = "A between 1 and 1"; // + Expression result = PARSER.parseExpression(expression); + Assertions.assertInstanceOf(EqualTo.class, result); + + expression = "A between 1 and 2"; + result = PARSER.parseExpression(expression); + Assertions.assertInstanceOf(And.class, result); + } +} From b744a28dde31a6e168c0ccc609573f198dfddcf3 Mon Sep 17 00:00:00 2001 From: englefly Date: Thu, 7 Nov 2024 08:50:43 +0800 Subject: [PATCH 2/3] support date convert --- .../expression/ExpressionOptimization.java | 4 +- .../expression/rules/BetweenToEqual.java | 115 ++++++++++++++++++ .../rules/expression/SimplifyRangeTest.java | 1 + .../between_to_equal/betweenToEqual.groovy | 46 +++++++ 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BetweenToEqual.java create mode 100644 regression-test/suites/nereids_rules_p0/between_to_equal/betweenToEqual.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java index 0ae179489aceb1..5d45b2430c42b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionOptimization.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.expression; import org.apache.doris.nereids.rules.expression.rules.ArrayContainToArrayOverlap; +import org.apache.doris.nereids.rules.expression.rules.BetweenToEqual; import org.apache.doris.nereids.rules.expression.rules.CaseWhenToIf; import org.apache.doris.nereids.rules.expression.rules.DateFunctionRewrite; import org.apache.doris.nereids.rules.expression.rules.DistinctPredicatesRule; @@ -53,7 +54,8 @@ public class ExpressionOptimization extends ExpressionRewrite { CaseWhenToIf.INSTANCE, TopnToMax.INSTANCE, NullSafeEqualToEqual.INSTANCE, - LikeToEqualRewrite.INSTANCE + LikeToEqualRewrite.INSTANCE, + BetweenToEqual.INSTANCE ) ); private static final ExpressionRuleExecutor EXECUTOR = new ExpressionRuleExecutor(OPTIMIZE_REWRITE_RULES); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BetweenToEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BetweenToEqual.java new file mode 100644 index 00000000000000..d1279fdfda55d6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BetweenToEqual.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; +import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; +import org.apache.doris.nereids.trees.expressions.And; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +/** + * f(A, B) between 1 and 1 => f(A, B) = 1 + * + */ +public class BetweenToEqual implements ExpressionPatternRuleFactory { + + public static BetweenToEqual INSTANCE = new BetweenToEqual(); + + @Override + public List> buildRules() { + return ImmutableList.of( + matchesType(And.class).then(BetweenToEqual::rewriteBetweenToEqual) + ); + } + + private static Expression rewriteBetweenToEqual(And and) { + List conjuncts = ExpressionUtils.extractConjunction(and); + Map> betweenCandidate = Maps.newHashMap(); + for (Expression conj : conjuncts) { + if (isCandidate(conj)) { + conj = normalizeCandidate((ComparisonPredicate) conj); + Expression varPart = conj.child(0); + betweenCandidate.computeIfAbsent(varPart, k -> Lists.newArrayList()); + betweenCandidate.get(varPart).add((ComparisonPredicate) conj); + } + } + List equals = Lists.newArrayList(); + List equalsKey = Lists.newArrayList(); + for (Expression varPart : betweenCandidate.keySet()) { + List candidates = betweenCandidate.get(varPart); + if (candidates.size() == 2 && greaterEqualAndLessEqual(candidates.get(0), candidates.get(1))) { + if (candidates.get(0).child(1).equals(candidates.get(1).child(1))) { + equals.add(new EqualTo(candidates.get(0).child(0), candidates.get(0).child(1))); + equalsKey.add(candidates.get(0).child(0)); + } + } + } + if (equals.isEmpty()) { + return null; + } else { + List newConjuncts = Lists.newArrayList(equals); + for (Expression conj : conjuncts) { + if (isCandidate(conj)) { + conj = normalizeCandidate((ComparisonPredicate) conj); + if (equalsKey.contains(conj.child(0))) { + continue; + } + } + newConjuncts.add(conj); + } + return ExpressionUtils.and(newConjuncts); + } + } + + // A >= a + // A <= a + // A is expr, a is literal + private static boolean isCandidate(Expression expr) { + if (expr instanceof GreaterThanEqual || expr instanceof LessThanEqual) { + return expr.child(0) instanceof Literal && !(expr.child(1) instanceof Literal) + || expr.child(1) instanceof Literal && !(expr.child(0) instanceof Literal); + } + return false; + } + + private static Expression normalizeCandidate(ComparisonPredicate expr) { + if (expr.child(1) instanceof Literal) { + return expr; + } else { + return expr.withChildren(expr.child(1), expr.child(0)); + } + } + + private static boolean greaterEqualAndLessEqual(ComparisonPredicate cmp1, ComparisonPredicate cmp2) { + return cmp1 instanceof GreaterThanEqual && cmp2 instanceof LessThanEqual + || (cmp1 instanceof LessThanEqual && cmp2 instanceof GreaterThanEqual); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java index 1f622619dd6266..329efee77a8f57 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java @@ -359,6 +359,7 @@ public void testSimplifyDateTime() { "(CA is null and null) OR CB < timestamp '2024-01-05 00:50:00'"); } + @Test private void assertRewrite(String expression, String expected) { Map mem = Maps.newHashMap(); Expression needRewriteExpression = replaceUnboundSlot(PARSER.parseExpression(expression), mem); diff --git a/regression-test/suites/nereids_rules_p0/between_to_equal/betweenToEqual.groovy b/regression-test/suites/nereids_rules_p0/between_to_equal/betweenToEqual.groovy new file mode 100644 index 00000000000000..9ce93e3fac39ee --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/between_to_equal/betweenToEqual.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("between_to_equal") { + sql """ + create table datebetween ( + guid int, + dt DATE, + first_visit_time varchar + )Engine=Olap + DUPLICATE KEY(guid) + distributed by hash(dt) buckets 3 + properties('replication_num' = '1'); + + insert into datebetween values (1, '2021-01-01', 'abc'); + """ + explain { + sql " select * from datebetween where dt between '2021-01-01' and '2021-01-01 11:11:11';" + contains("PREDICATES: (dt[#1] = '2021-01-01')"); + } + + explain { + sql " select * from datebetween where dt between '2021-01-01' and '2021-01-01 11:11:11' and dt < '2024-12-01';" + contains("PREDICATES: (dt[#1] = '2021-01-01')") + } + + explain { + sql "select * from datebetween where dt between '2021-01-01' and '2021-01-01 11:11:11' and dt < '2024-12-01' or guid =1;" + contains("PREDICATES: ((dt[#1] = '2021-01-01') OR (guid[#0] = 1))") + } + +} From e49b92feaed0e0c514145f76bda111985c79a0d2 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 11 Nov 2024 16:33:17 +0800 Subject: [PATCH 3/3] fix bug in TimestampArithmetic.equals --- .../doris/nereids/trees/expressions/TimestampArithmetic.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java index 888a1a0869ccbb..d3e326fa48a574 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/TimestampArithmetic.java @@ -160,6 +160,7 @@ public boolean equals(Object o) { } TimestampArithmetic other = (TimestampArithmetic) o; return Objects.equals(funcName, other.funcName) && Objects.equals(timeUnit, other.timeUnit) + && op.equals(other.op) && Objects.equals(left(), other.left()) && Objects.equals(right(), other.right()); } }