From c0b34f5741d2aed543abc3f529b49b5bc052b4ef Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Thu, 25 Jul 2024 11:36:11 +0800 Subject: [PATCH] [opt](Nereids) add where Null rule to create empty relation as where false (#38135) explain shape plan select * from table2 where Null; explain shape plan select * from table2 where false; in this case, null literal can be regard as same as false literal --- .../rules/rewrite/EliminateFilter.java | 8 +- .../rules/rewrite/EliminateFilterTest.java | 12 ++ .../data/empty_relation/eliminate_empty.out | 67 ++++++++++++ .../empty_relation/eliminate_empty.groovy | 103 +++++++++++++++++- .../test_simplify_comparison.groovy | 20 +--- 5 files changed, 187 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java index ef9e418f58d185..1413faf3bb068d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateFilter.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -43,12 +44,13 @@ public class EliminateFilter implements RewriteRuleFactory { @Override public List buildRules() { return ImmutableList.of(logicalFilter().when( - filter -> ExpressionUtils.containsType(filter.getConjuncts(), BooleanLiteral.class)) + filter -> ExpressionUtils.containsType(filter.getConjuncts(), BooleanLiteral.class) + || ExpressionUtils.containsType(filter.getConjuncts(), NullLiteral.class)) .thenApply(ctx -> { LogicalFilter filter = ctx.root; ImmutableSet.Builder newConjuncts = ImmutableSet.builder(); for (Expression expression : filter.getConjuncts()) { - if (expression == BooleanLiteral.FALSE) { + if (expression == BooleanLiteral.FALSE || expression.isNullLiteral()) { return new LogicalEmptyRelation(ctx.statementContext.getNextRelationId(), filter.getOutput()); } else if (expression != BooleanLiteral.TRUE) { @@ -75,7 +77,7 @@ public List buildRules() { Expression newExpr = ExpressionUtils.replace(expression, replaceMap); Expression foldExpression = FoldConstantRule.evaluate(newExpr, context); - if (foldExpression == BooleanLiteral.FALSE) { + if (foldExpression == BooleanLiteral.FALSE || expression.isNullLiteral()) { return new LogicalEmptyRelation( ctx.statementContext.getNextRelationId(), filter.getOutput()); } else if (foldExpression != BooleanLiteral.TRUE) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java index cbdc95db559e10..d3d1316eaaca74 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateFilterTest.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.LogicalPlanBuilder; @@ -50,6 +51,17 @@ void testEliminateFilterFalse() { .matches(logicalEmptyRelation()); } + @Test + void testEliminateFilterNull() { + LogicalPlan filterNull = new LogicalPlanBuilder(scan1) + .filter(NullLiteral.INSTANCE) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), filterNull) + .applyTopDown(new EliminateFilter()) + .matches(logicalEmptyRelation()); + } + @Test void testEliminateFilterTrue() { LogicalPlan filterTrue = new LogicalPlanBuilder(scan1) diff --git a/regression-test/data/empty_relation/eliminate_empty.out b/regression-test/data/empty_relation/eliminate_empty.out index ca0d2a4641a571..56917b8042d436 100644 --- a/regression-test/data/empty_relation/eliminate_empty.out +++ b/regression-test/data/empty_relation/eliminate_empty.out @@ -70,6 +70,73 @@ PhysicalResultSink -- !except_empty_data -- +-- !null_join -- +PhysicalResultSink +--PhysicalEmptyRelation + +-- !null_explain_union_empty_data -- +PhysicalResultSink +--PhysicalDistribute[DistributionSpecGather] +----hashAgg[LOCAL] +------PhysicalProject +--------PhysicalOlapScan[nation] + +-- !null_union_empty_data -- +1 + +-- !null_explain_union_empty_empty -- +PhysicalResultSink +--PhysicalEmptyRelation + +-- !null_union_empty_empty -- + +-- !null_union_emtpy_onerow -- +10 + +-- !null_explain_intersect_data_empty -- +PhysicalResultSink +--PhysicalEmptyRelation + +-- !null_explain_intersect_empty_data -- +PhysicalResultSink +--PhysicalEmptyRelation + +-- !null_explain_except_data_empty -- +PhysicalResultSink +--PhysicalDistribute[DistributionSpecGather] +----PhysicalProject +------hashAgg[LOCAL] +--------PhysicalProject +----------PhysicalOlapScan[nation] + +-- !null_explain_except_data_empty_data -- +PhysicalResultSink +--PhysicalDistribute[DistributionSpecGather] +----PhysicalExcept +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------PhysicalOlapScan[nation] +------PhysicalDistribute[DistributionSpecHash] +--------PhysicalProject +----------filter(( not (n_nationkey = 1))) +------------PhysicalOlapScan[nation] + +-- !null_except_data_empty_data -- +1 + +-- !null_explain_except_empty_data -- +PhysicalResultSink +--PhysicalEmptyRelation + +-- !null_intersect_data_empty -- + +-- !null_intersect_empty_data -- + +-- !null_except_data_empty -- +1 + +-- !null_except_empty_data -- + -- !prune_partition1 -- PhysicalResultSink --PhysicalEmptyRelation diff --git a/regression-test/suites/empty_relation/eliminate_empty.groovy b/regression-test/suites/empty_relation/eliminate_empty.groovy index ffe3fe17cd59e4..af3ec156e08f36 100644 --- a/regression-test/suites/empty_relation/eliminate_empty.groovy +++ b/regression-test/suites/empty_relation/eliminate_empty.groovy @@ -131,6 +131,107 @@ suite("eliminate_empty") { select r_regionkey from region where false except select n_nationkey from nation """ + qt_null_join """ + explain shape plan + select * + from + nation + join + (select * from region where Null) R + """ + + qt_null_explain_union_empty_data """ + explain shape plan + select * + from (select n_nationkey from nation union select r_regionkey from region where Null) T + """ + qt_null_union_empty_data """ + select * + from (select n_nationkey from nation union select r_regionkey from region where Null) T + """ + + qt_null_explain_union_empty_empty """ + explain shape plan + select * + from ( + select n_nationkey from nation where Null + union + select r_regionkey from region where Null + ) T + """ + qt_null_union_empty_empty """ + select * + from ( + select n_nationkey from nation where Null + union + select r_regionkey from region where Null + ) T + """ + qt_null_union_emtpy_onerow """ + select * + from ( + select n_nationkey from nation where Null + union + select 10 + union + select 10 + )T + """ + + qt_null_explain_intersect_data_empty """ + explain shape plan + select n_nationkey from nation intersect select r_regionkey from region where Null + """ + + qt_null_explain_intersect_empty_data """ + explain shape plan + select r_regionkey from region where Null intersect select n_nationkey from nation + """ + + qt_null_explain_except_data_empty """ + explain shape plan + select n_nationkey from nation except select r_regionkey from region where Null + """ + + qt_null_explain_except_data_empty_data """ + explain shape plan + select n_nationkey from nation + except + select r_regionkey from region where Null + except + select n_nationkey from nation where n_nationkey != 1; + """ + + qt_null_except_data_empty_data """ + select n_nationkey from nation + except + select r_regionkey from region where Null + except + select n_nationkey from nation where n_nationkey != 1; + """ + + qt_null_explain_except_empty_data """ + explain shape plan + select r_regionkey from region where Null except select n_nationkey from nation + """ + + + qt_null_intersect_data_empty """ + select n_nationkey from nation intersect select r_regionkey from region where Null + """ + + qt_null_intersect_empty_data """ + select r_regionkey from region where Null intersect select n_nationkey from nation + """ + + qt_null_except_data_empty """ + select n_nationkey from nation except select r_regionkey from region where Null + """ + + qt_null_except_empty_data """ + select r_regionkey from region where Null except select n_nationkey from nation + """ + sql """ drop table if exists eliminate_partition_prune; """ @@ -218,4 +319,4 @@ suite("eliminate_empty") { sql """drop table if exists table_5_undef_partitions2_keys3""" sql """drop table if exists table_10_undef_partitions2_keys3""" } -} \ No newline at end of file +} diff --git a/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy b/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy index a5b3ef28e4dea9..86b1402c6824d5 100644 --- a/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_simplify_comparison.groovy @@ -89,15 +89,6 @@ suite("test_simplify_comparison") { contains "CAST" } - explain { - sql "verbose select * from simple_test_table_t where a = cast(1.1 as double) and b = cast(1.1 as double) and c = cast(1.1 as double) and d = cast(1.1 as double);" - contains "a[#0] IS NULL" - contains "b[#1] IS NULL" - contains "c[#2] IS NULL" - contains "d[#3] IS NULL" - contains "AND NULL" - } - explain { sql "verbose select * from simple_test_table_t where e = cast(1.1 as double);" contains "CAST(e[#4] AS DOUBLE) = 1.1" @@ -205,15 +196,6 @@ suite("test_simplify_comparison") { contains "CAST" } - explain { - sql "verbose select * from simple_test_table_t where a = 1.1 and b = 1.1 and c = 1.1 and d = 1.1;" - contains "a[#0] IS NULL" - contains "b[#1] IS NULL" - contains "c[#2] IS NULL" - contains "d[#3] IS NULL" - contains "AND NULL" - } - explain { sql "verbose select * from simple_test_table_t where e = 1.1;" contains "CAST(e[#4] AS DOUBLE) = 1.1" @@ -272,4 +254,4 @@ suite("test_simplify_comparison") { } qt_select1 """select * from simple_test_table_t where cast(a as decimal(5,1)) = 10.0;""" qt_select2 """select a.col1, cast(a.col1 as decimal(7,2)) col3, case when a.col1 is null then 15 when cast(a.col1 as decimal(7,2)) < -99997.99 then 18 when cast(a.col1 as decimal(7,2)) < 1.001 then 3 else -55 end col2 from (select 1 as col1) a;""" -} \ No newline at end of file +}