From 56dd81892d23737b9a791dfd90b7bc2e68f070f2 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Fri, 25 Aug 2023 17:58:19 +0800 Subject: [PATCH 1/3] fix divide double can not match mv --- .../AbstractSelectMaterializedIndexRule.java | 4 +- .../expressions/literal/DoubleLiteral.java | 9 ++++ .../trees/plans/logical/LogicalProject.java | 1 - .../mv_p0/test_dup_mv_div/test_dup_mv_div.out | 13 +++++ .../test_dup_mv_div/test_dup_mv_div.groovy | 52 +++++++++++++++++++ 5 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out create mode 100644 regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index e91c685fc2182e..3e02be52da2a0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -148,7 +148,7 @@ protected static Expression removeCastAndAlias(Expression expression) { } protected static boolean containsAllColumn(Expression expression, Set mvColumnNames) { - if (mvColumnNames.contains(Column.getNameWithoutMvPrefix(expression.toSql()))) { + if (mvColumnNames.contains(expression.toSql())) { return true; } if (expression.children().isEmpty()) { @@ -412,7 +412,7 @@ protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { } } if (!isPushed) { - if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) { + if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) { mvNameToMvSlot.put(normalizeName( org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvSlot.getName())), mvSlot); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java index 34b8ca36fbc8a5..bdd26460c06b96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DoubleLiteral.java @@ -23,6 +23,8 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DoubleType; +import java.text.NumberFormat; + /** * Double literal */ @@ -49,4 +51,11 @@ public R accept(ExpressionVisitor visitor, C context) { public LiteralExpr toLegacyLiteral() { return new FloatLiteral(value, Type.DOUBLE); } + + @Override + public String toString() { + NumberFormat nf = NumberFormat.getInstance(); + nf.setGroupingUsed(false); + return nf.format(value); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index ba4e4c6d7ae989..6e3ebf9d8f915e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -231,7 +231,6 @@ public JSONObject toJson() { } public LogicalProject readFromJson(JSONObject logicalProject) { - return new LogicalProject<>(ImmutableList.of(new UnboundStar(ImmutableList.of())), null, null, isDistinct); } diff --git a/regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out b/regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out new file mode 100644 index 00000000000000..64c881ed10b021 --- /dev/null +++ b/regression-test/data/mv_p0/test_dup_mv_div/test_dup_mv_div.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +-4 -4 -4 4 +1 1 1 1 +2 2 2 b +3 -3 \N c + +-- !select_mv -- +-4 -4.0 +1 1.0 +2 2.0 +3 -3.0 + diff --git a/regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy b/regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy new file mode 100644 index 00000000000000..17d2fbf2e5816e --- /dev/null +++ b/regression-test/suites/mv_p0/test_dup_mv_div/test_dup_mv_div.groovy @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_dup_mv_div") { + sql """set enable_nereids_planner=true""" + sql """SET enable_fallback_to_original_planner=false""" + sql """ DROP TABLE IF EXISTS d_table; """ + + sql """ + create table d_table( + k1 int null, + k2 int not null, + k3 bigint null, + k4 varchar(100) null + ) + duplicate key (k1,k2,k3) + distributed BY hash(k1) buckets 3 + properties("replication_num" = "1"); + """ + + sql "insert into d_table select 1,1,1,'1';" + sql "insert into d_table select 2,2,2,'b';" + sql "insert into d_table select 3,-3,null,'c';" + + createMV ("create materialized view kdiv as select k1,k2/1 from d_table;") + + sql "insert into d_table select -4,-4,-4,'4';" + + qt_select_star "select * from d_table order by k1;" + + explain { + sql("select k1,k2/1 from d_table order by k1;") + contains "(kdiv)" + } + qt_select_mv "select k1,k2/1 from d_table order by k1;" +} From 5e37a05ea7879a7cc402bf2e51426e106a7026f5 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 28 Aug 2023 13:39:34 +0800 Subject: [PATCH 2/3] fix --- .../AbstractSelectMaterializedIndexRule.java | 5 ++++- .../expression/SimplifyArithmeticRuleTest.java | 18 +++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 3e02be52da2a0e..58b3d105747cb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -400,7 +400,7 @@ protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { for (Slot mvSlot : mvPlan.getOutputByIndex(mvPlan.getSelectedIndexId())) { boolean isPushed = false; for (Slot baseSlot : mvPlan.getOutput()) { - if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumnAggregate(mvSlot.getName())) { + if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) { continue; } if (baseSlot.toSql().equalsIgnoreCase( @@ -570,6 +570,9 @@ public Expression visitSlotReference(SlotReference slotReference, Void context) if (baseSlotToMvSlot.containsKey(slotReference)) { return baseSlotToMvSlot.get(slotReference); } + if (mvNameToMvSlot.containsKey(slotReference.toSql())) { + return mvNameToMvSlot.get(slotReference.toSql()); + } return slotReference; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java index 8b9f48010aae6e..567073254a9376 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyArithmeticRuleTest.java @@ -36,7 +36,7 @@ public void testSimplifyArithmetic() { assertRewriteAfterTypeCoercion("IA", "IA"); assertRewriteAfterTypeCoercion("IA + 1", "IA + 1"); assertRewriteAfterTypeCoercion("IA + IB", "IA + IB"); - assertRewriteAfterTypeCoercion("1 * 3 / IA", "(3.0 / cast(IA as DOUBLE))"); + assertRewriteAfterTypeCoercion("1 * 3 / IA", "(3 / cast(IA as DOUBLE))"); assertRewriteAfterTypeCoercion("1 - IA", "1 - IA"); assertRewriteAfterTypeCoercion("1 + 1", "2"); assertRewriteAfterTypeCoercion("IA + 2 - 1", "IA + 1"); @@ -44,12 +44,12 @@ public void testSimplifyArithmetic() { assertRewriteAfterTypeCoercion("IA + 2 - ((1 - IB) - (3 + IC))", "IA + IB + IC + 4"); assertRewriteAfterTypeCoercion("IA * IB + 2 - IC * 2", "(IA * IB) - (IC * 2) + 2"); assertRewriteAfterTypeCoercion("IA * IB", "IA * IB"); - assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as DOUBLE) / 1.0"); - assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4.0"); - assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4.0"); - assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as DOUBLE) * cast(IB as DOUBLE) / 1.0"); - assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2.0"); - assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4", "(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) / 4.0)"); + assertRewriteAfterTypeCoercion("IA * IB / 2 * 2", "cast((IA * IB) as DOUBLE) / 1"); + assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4"); + assertRewriteAfterTypeCoercion("IA * IB / (2 * 2)", "cast((IA * IB) as DOUBLE) / 4"); + assertRewriteAfterTypeCoercion("IA * (IB / 2) * 2)", "cast(IA as DOUBLE) * cast(IB as DOUBLE) / 1"); + assertRewriteAfterTypeCoercion("IA * (IB / 2) * (IC + 1))", "cast(IA as DOUBLE) * cast(IB as DOUBLE) * cast((IC + 1) as DOUBLE) / 2"); + assertRewriteAfterTypeCoercion("IA * IB / 2 / IC * 2 * ID / 4", "(((cast((IA * IB) as DOUBLE) / cast(IC as DOUBLE)) * cast(ID as DOUBLE)) / 4)"); } @Test @@ -86,8 +86,8 @@ public void testSimplifyArithmeticComparison() { assertRewriteAfterTypeCoercion("IA + 1 > IB", "cast(IA as BIGINT) > (cast(IB as BIGINT) - 1)"); assertRewriteAfterTypeCoercion("IA + 1 > IB * IC", "cast(IA as BIGINT) > ((IB * IC) - 1)"); assertRewriteAfterTypeCoercion("IA * ID > IB * IC", "IA * ID > IB * IC"); - assertRewriteAfterTypeCoercion("IA * ID / 2 > IB * IC", "cast((IA * ID) as DOUBLE) > cast((IB * IC) as DOUBLE) * 2.0"); - assertRewriteAfterTypeCoercion("IA * ID / -2 > IB * IC", "cast((IB * IC) as DOUBLE) * -2.0 > cast((IA * ID) as DOUBLE)"); + assertRewriteAfterTypeCoercion("IA * ID / 2 > IB * IC", "cast((IA * ID) as DOUBLE) > cast((IB * IC) as DOUBLE) * 2"); + assertRewriteAfterTypeCoercion("IA * ID / -2 > IB * IC", "cast((IB * IC) as DOUBLE) * -2 > cast((IA * ID) as DOUBLE)"); } } From d4fcbb0bdab42fe479153152dc56bf8ac256a61a Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 28 Aug 2023 15:19:58 +0800 Subject: [PATCH 3/3] fix --- .../data/nereids_tpcds_shape_sf100_p0/shape/query11.out | 2 +- .../data/nereids_tpcds_shape_sf100_p0/shape/query39.out | 2 +- .../data/nereids_tpcds_shape_sf100_p0/shape/query73.out | 2 +- .../data/nereids_tpcds_shape_sf100_p0/shape/query74.out | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out index 61dc688e5f5cdd..6e395565dd43ed 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query11.out @@ -42,7 +42,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute --------PhysicalTopN ----------PhysicalProject -------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0.0 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0.0 END) +------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id)(CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0 END > CASE WHEN (year_total > 0.00) THEN (cast(year_total as DOUBLE) / cast(year_total as DOUBLE)) ELSE 0 END) --------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id) ----------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id) ------------------PhysicalDistribute diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out index 0fe8176d6f271f..3a2ff5eb3dc3e6 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out @@ -3,7 +3,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) ----PhysicalProject -------filter((CASE WHEN (mean = 0.0) THEN 0.0 ELSE (stdev / mean) END > 1.0)) +------filter((CASE WHEN (mean = 0) THEN 0 ELSE (stdev / mean) END > 1)) --------hashAgg[GLOBAL] ----------PhysicalDistribute ------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out index 61342aa658f34b..34a39fb42f31b3 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query73.out @@ -25,7 +25,7 @@ PhysicalResultSink ------------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalDistribute ------------------------------PhysicalProject ---------------------------------filter(((cast(hd_buy_potential as VARCHAR(*)) = '501-1000') OR (cast(hd_buy_potential as VARCHAR(*)) = 'Unknown'))(household_demographics.hd_vehicle_count > 0)(CASE WHEN (hd_vehicle_count > 0) THEN (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) ELSE NULL END > 1.0)) +--------------------------------filter(((cast(hd_buy_potential as VARCHAR(*)) = '501-1000') OR (cast(hd_buy_potential as VARCHAR(*)) = 'Unknown'))(household_demographics.hd_vehicle_count > 0)(CASE WHEN (hd_vehicle_count > 0) THEN (cast(hd_dep_count as DOUBLE) / cast(hd_vehicle_count as DOUBLE)) ELSE NULL END > 1)) ----------------------------------PhysicalOlapScan[household_demographics] --------------------------PhysicalDistribute ----------------------------PhysicalProject diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out index 9cd4ce49056dfd..370d84a76438e1 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query74.out @@ -42,12 +42,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------PhysicalDistribute --------PhysicalTopN ----------PhysicalProject -------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0.0) THEN (year_total / year_total) ELSE NULL END) +------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_firstyear.customer_id)(CASE WHEN (year_total > 0) THEN (year_total / year_total) ELSE NULL END > CASE WHEN (year_total > 0) THEN (year_total / year_total) ELSE NULL END) --------------hashJoin[INNER_JOIN](t_s_firstyear.customer_id = t_w_secyear.customer_id) ----------------hashJoin[INNER_JOIN](t_s_secyear.customer_id = t_s_firstyear.customer_id) ------------------PhysicalDistribute --------------------PhysicalProject -----------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0.0)) +----------------------filter((t_s_firstyear.year = 1999)(t_s_firstyear.sale_type = 's')(t_s_firstyear.year_total > 0)) ------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ------------------PhysicalDistribute --------------------PhysicalProject @@ -59,6 +59,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------PhysicalDistribute ----------------PhysicalProject -------------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0.0)(t_w_firstyear.sale_type = 'w')) +------------------filter((t_w_firstyear.year = 1999)(t_w_firstyear.year_total > 0)(t_w_firstyear.sale_type = 'w')) --------------------PhysicalCteConsumer ( cteId=CTEId#0 )