From 19a35422b4a6e27469cc956a3359b9c24f950217 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 30 Jul 2024 18:47:28 +0800 Subject: [PATCH 1/8] support eliminate outer join by match expression --- .../rules/analysis/CheckAfterRewrite.java | 25 ++++++++++++------- .../rules/FoldConstantRuleOnFE.java | 11 ++++++++ .../data/nereids_syntax_p0/match.out | 8 ++++++ .../suites/nereids_syntax_p0/match.groovy | 11 +++++++- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index df8ec64fc2e1ff..7831d102ac82bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -184,16 +184,23 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { private void checkMatchIsUsedCorrectly(Plan plan) { for (Expression expression : plan.getExpressions()) { - if (expression instanceof Match) { - if (plan instanceof LogicalFilter && (plan.child(0) instanceof LogicalOlapScan - || plan.child(0) instanceof LogicalDeferMaterializeOlapScan)) { - return; - } else { - throw new AnalysisException(String.format( - "Not support match in %s in plan: %s, only support in olapScan filter", - plan.child(0), plan)); + expression.foreach(expr -> { + if (expr instanceof Match) { + boolean valid = plan instanceof LogicalFilter + && (plan.child(0) instanceof LogicalOlapScan + || plan.child(0) instanceof LogicalDeferMaterializeOlapScan); + if (!valid) { + if (plan instanceof LogicalFilter) { + throw new AnalysisException(String.format( + "Not support match in %s in plan: %s, only support in olapScan filter", + plan.child(0), plan)); + } else { + throw new AnalysisException(String.format( + "Not support match in plan: %s, only support in olapScan filter", plan)); + } + } } - } + }); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java index d7627d698d6134..fdd3b02e6fd483 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; @@ -190,6 +191,16 @@ public Expression visitLiteral(Literal literal, ExpressionRewriteContext context return literal; } + @Override + public Expression visitMatch(Match match, ExpressionRewriteContext context) { + match = rewriteChildren(match, context); + Optional checkedExpr = preProcess(match); + if (checkedExpr.isPresent()) { + return checkedExpr.get(); + } + return super.visitMatch(match, context); + } + @Override public Expression visitEncryptKeyRef(EncryptKeyRef encryptKeyRef, ExpressionRewriteContext context) { String dbName = encryptKeyRef.getDbName(); diff --git a/regression-test/data/nereids_syntax_p0/match.out b/regression-test/data/nereids_syntax_p0/match.out index 3664b141e4bbb0..efb8baa2fcc4b4 100644 --- a/regression-test/data/nereids_syntax_p0/match.out +++ b/regression-test/data/nereids_syntax_p0/match.out @@ -109,3 +109,11 @@ li ba li liuliu -- !match_phrase_7 -- +-- !match_join -- +li sisi 11 grade 6 li ba li liuliu zhang san yi 11 grade 5 zhang yi chen san learn makes me happy +san zhang 10 grade 5 san zhang 10 grade 5 +san zhang 10 grade 5 zhang san 10 grade 5 zhang yi chen san Class activists +zhang san 10 grade 5 zhang yi chen san Class activists san zhang 10 grade 5 +zhang san 10 grade 5 zhang yi chen san Class activists zhang san 10 grade 5 zhang yi chen san Class activists +zhang san yi 11 grade 5 zhang yi chen san learn makes me happy zhang san yi 11 grade 5 zhang yi chen san learn makes me happy + diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index edc4f8643910db..2c80d7eee294be 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_nereids_match_select") { +suite("match") { sql """ SET enable_nereids_planner=true """ @@ -172,5 +172,14 @@ suite("test_nereids_match_select") { order_qt_match_phrase_7 """ SELECT * FROM test_nereids_match_select WHERE name match_phrase 'zhang' and selfComment match_phrase 'want go outside'; """ + + order_qt_match_join """ + select * + from test_nereids_match_select a + left join + test_nereids_match_select b + on a.age = b.age + where b.name match_any 'zhang' + """ } From e577ea47aae483f5878d5f5b6b663a72464a5bd3 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Tue, 30 Jul 2024 20:08:47 +0800 Subject: [PATCH 2/8] fix --- .../rules/analysis/CheckAfterRewrite.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 7831d102ac82bd..e193c5fc4938de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; @@ -39,9 +38,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.Generate; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; @@ -64,7 +60,6 @@ public Rule build() { checkAllSlotReferenceFromChildren(plan); checkUnexpectedExpression(plan); checkMetricTypeIsUsedCorrectly(plan); - checkMatchIsUsedCorrectly(plan); return null; }).toRule(RuleType.CHECK_ANALYSIS); } @@ -181,26 +176,4 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) { }); } } - - private void checkMatchIsUsedCorrectly(Plan plan) { - for (Expression expression : plan.getExpressions()) { - expression.foreach(expr -> { - if (expr instanceof Match) { - boolean valid = plan instanceof LogicalFilter - && (plan.child(0) instanceof LogicalOlapScan - || plan.child(0) instanceof LogicalDeferMaterializeOlapScan); - if (!valid) { - if (plan instanceof LogicalFilter) { - throw new AnalysisException(String.format( - "Not support match in %s in plan: %s, only support in olapScan filter", - plan.child(0), plan)); - } else { - throw new AnalysisException(String.format( - "Not support match in plan: %s, only support in olapScan filter", plan)); - } - } - } - }); - } - } } From 7bfe0ff07aff8d973978722d701be837744509bb Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Wed, 31 Jul 2024 10:36:55 +0800 Subject: [PATCH 3/8] support eliminate outer join by match expression --- .../suites/nereids_syntax_p0/match.groovy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index 2c80d7eee294be..b49d88b105ebbb 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -173,6 +173,19 @@ suite("match") { SELECT * FROM test_nereids_match_select WHERE name match_phrase 'zhang' and selfComment match_phrase 'want go outside'; """ + explain { + sql """ + select * + from test_nereids_match_select a + left join + test_nereids_match_select b + on a.age = b.age + where b.name match_any 'zhang' + """ + + contains("INNER JOIN") + } + order_qt_match_join """ select * from test_nereids_match_select a From 1a1b86b5ca9046ccdf73ff0045b054fea19c3231 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Wed, 31 Jul 2024 11:01:39 +0800 Subject: [PATCH 4/8] fix --- .../doris/nereids/rules/expression/rules/PartitionPruner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index b0b45077dcc21e..efe12f38cd74e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -130,7 +130,7 @@ public static List prune(List partitionSlots, Expression partitionPr partitionPredicate, new ExpressionRewriteContext(cascadesContext)); if (BooleanLiteral.TRUE.equals(partitionPredicate)) { return Utils.fastToImmutableList(idToPartitions.keySet()); - } else if (Boolean.FALSE.equals(partitionPredicate) || partitionPredicate.isNullLiteral()) { + } else if (BooleanLiteral.FALSE.equals(partitionPredicate) || partitionPredicate.isNullLiteral()) { return ImmutableList.of(); } From 8fff252ec2f42293b26c67adfabdc1a575b8313d Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 5 Aug 2024 15:31:32 +0800 Subject: [PATCH 5/8] add show variable --- regression-test/suites/nereids_syntax_p0/match.groovy | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index b49d88b105ebbb..57e1dc82b45ae4 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -1,3 +1,5 @@ +import java.util.stream.Collectors + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -186,6 +188,12 @@ suite("match") { contains("INNER JOIN") } + def variables = sql "show variables" + def variableString = variables.stream() + .map { it.toString() } + .collect(Collectors.joining("\n")) + logger.info("Variables:\n${variableString}") + order_qt_match_join """ select * from test_nereids_match_select a From 3ac3b060f1c04c30b7a111cabc965011587ef419 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 5 Aug 2024 17:09:44 +0800 Subject: [PATCH 6/8] add show variable --- .../suites/nereids_syntax_p0/match.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index 57e1dc82b45ae4..bb63d30a043f28 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -175,6 +175,12 @@ suite("match") { SELECT * FROM test_nereids_match_select WHERE name match_phrase 'zhang' and selfComment match_phrase 'want go outside'; """ + def variables = sql "show variables" + def variableString = variables.stream() + .map { it.toString() } + .collect(Collectors.joining("\n")) + logger.info("Variables:\n${variableString}") + explain { sql """ select * @@ -188,12 +194,6 @@ suite("match") { contains("INNER JOIN") } - def variables = sql "show variables" - def variableString = variables.stream() - .map { it.toString() } - .collect(Collectors.joining("\n")) - logger.info("Variables:\n${variableString}") - order_qt_match_join """ select * from test_nereids_match_select a From 5f0896bd4f65ade4299490b06e8eefdaa9c7db44 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 5 Aug 2024 19:09:40 +0800 Subject: [PATCH 7/8] set enable_fold_constant_by_be=false --- regression-test/suites/nereids_syntax_p0/match.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index bb63d30a043f28..6a68d2394400b2 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -181,6 +181,8 @@ suite("match") { .collect(Collectors.joining("\n")) logger.info("Variables:\n${variableString}") + sql "set enable_fold_constant_by_be=false" + explain { sql """ select * From 74f296b91de361281e16d36a300ed19a7f9ffb7a Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 5 Aug 2024 19:11:14 +0800 Subject: [PATCH 8/8] set enable_fold_constant_by_be=false --- regression-test/suites/nereids_syntax_p0/match.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/nereids_syntax_p0/match.groovy b/regression-test/suites/nereids_syntax_p0/match.groovy index 6a68d2394400b2..4505460ac27c5f 100644 --- a/regression-test/suites/nereids_syntax_p0/match.groovy +++ b/regression-test/suites/nereids_syntax_p0/match.groovy @@ -1,5 +1,3 @@ -import java.util.stream.Collectors - // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -17,6 +15,8 @@ import java.util.stream.Collectors // specific language governing permissions and limitations // under the License. +import java.util.stream.Collectors + suite("match") { sql """ SET enable_nereids_planner=true