From 952adca731b1c3618e0d964f28d3ae1adbcb735a Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 6 Aug 2024 11:23:13 +0800 Subject: [PATCH] [enhancement](nereids) Support eliminate outer join by match expression (#38537) 1. enable run match expression outer of filter plan, e.g join conjunct 2. support eliminate outer join by match expression, if any arguments of match expression is null literal (cherry picked from commit 6b7f21c4de77d7ef1ef97192cef844d51d0268d1) --- .../rules/analysis/CheckAfterRewrite.java | 20 ----------- .../rules/FoldConstantRuleOnFE.java | 11 ++++++ .../expression/rules/PartitionPruner.java | 2 +- .../data/nereids_syntax_p0/match.out | 8 +++++ .../suites/nereids_syntax_p0/match.groovy | 34 ++++++++++++++++++- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index df8ec64fc2e1ff..e193c5fc4938de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; @@ -39,9 +38,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Generate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; @@ -64,7 +60,6 @@ public Rule build() { checkAllSlotReferenceFromChildren(plan); checkUnexpectedExpression(plan); checkMetricTypeIsUsedCorrectly(plan); - checkMatchIsUsedCorrectly(plan); return null; }).toRule(RuleType.CHECK_ANALYSIS); } @@ -181,19 +176,4 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { }); } } - - private void checkMatchIsUsedCorrectly(Plan plan) { - for (Expression expression : plan.getExpressions()) { - if (expression instanceof Match) { - if (plan instanceof LogicalFilter && (plan.child(0) instanceof LogicalOlapScan - || plan.child(0) instanceof LogicalDeferMaterializeOlapScan)) { - return; - } else { - throw new AnalysisException(String.format( - "Not support match in %s in plan: %s, only support in olapScan filter", - plan.child(0), plan)); - } - } - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index 31ed154da3d1b3..8c1b5538205764 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; @@ -190,6 +191,16 @@ public Expression visitLiteral(Literal literal, ExpressionRewriteContext context return literal; } + @Override + public Expression visitMatch(Match match, ExpressionRewriteContext context) { + match = rewriteChildren(match, context); + Optional checkedExpr = preProcess(match); + if (checkedExpr.isPresent()) { + return checkedExpr.get(); + } + return super.visitMatch(match, context); + } + @Override public Expression visitEncryptKeyRef(EncryptKeyRef encryptKeyRef, ExpressionRewriteContext context) { String dbName = encryptKeyRef.getDbName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index b0b45077dcc21e..efe12f38cd74e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -130,7 +130,7 @@ public static List prune(List partitionSlots, Expression partitionPr partitionPredicate, new ExpressionRewriteContext(cascadesContext)); if (BooleanLiteral.TRUE.equals(partitionPredicate)) { return Utils.fastToImmutableList(idToPartitions.keySet()); - } else if (Boolean.FALSE.equals(partitionPredicate) || partitionPredicate.isNullLiteral()) { + } else if (BooleanLiteral.FALSE.equals(partitionPredicate) || partitionPredicate.isNullLiteral()) { return ImmutableList.of(); } diff --git a/regression-test/data/nereids_syntax_p0/match.out b/regression-test/data/nereids_syntax_p0/match.out index 3664b141e4bbb0..efb8baa2fcc4b4 100644 --- a/regression-test/data/nereids_syntax_p0/match.out +++ b/regression-test/data/nereids_syntax_p0/match.out @@ -109,3 +109,11 @@ li ba li liuliu -- !match_phrase_7 -- +-- !match_join -- +li sisi 11 grade 6 li ba li liuliu zhang san yi 11 grade 5 zhang yi chen san learn makes me happy +san zhang 10 grade 5 san zhang 10 grade 5 +san zhang 10 grade 5 zhang san 10 grade 5 zhang yi chen san Class activists +zhang san 10 grade 5 zhang yi chen san Class activists san zhang 10 grade 5 +zhang san 10 grade 5 zhang yi chen san Class activists zhang san 10 grade 5 zhang yi chen san Class activists +zhang san yi 11 grade 5 zhang yi chen san learn makes me happy zhang san yi 11 grade 5 zhang yi chen san learn makes me happy + diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index edc4f8643910db..4505460ac27c5f 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -suite("test_nereids_match_select") { +import java.util.stream.Collectors + +suite("match") { sql """ SET enable_nereids_planner=true """ @@ -172,5 +174,35 @@ suite("test_nereids_match_select") { order_qt_match_phrase_7 """ SELECT * FROM test_nereids_match_select WHERE name match_phrase 'zhang' and selfComment match_phrase 'want go outside'; """ + + def variables = sql "show variables" + def variableString = variables.stream() + .map { it.toString() } + .collect(Collectors.joining("\n")) + logger.info("Variables:\n${variableString}") + + sql "set enable_fold_constant_by_be=false" + + explain { + sql """ + select * + from test_nereids_match_select a + left join + test_nereids_match_select b + on a.age = b.age + where b.name match_any 'zhang' + """ + + contains("INNER JOIN") + } + + order_qt_match_join """ + select * + from test_nereids_match_select a + left join + test_nereids_match_select b + on a.age = b.age + where b.name match_any 'zhang' + """ }