diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index d317b1e8738521..7dcb7c0c0084ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -44,11 +44,15 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectScanRule; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterJoinRule; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectJoinRule; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectScanRule; import org.apache.doris.nereids.rules.expression.ExpressionOptimization; import org.apache.doris.nereids.rules.implementation.AggregateStrategies; import org.apache.doris.nereids.rules.implementation.LogicalAssertNumRowsToPhysicalAssertNumRows; @@ -243,6 +247,10 @@ public class RuleSet { .add(MaterializedViewFilterAggregateRule.INSTANCE) .add(MaterializedViewProjectFilterAggregateRule.INSTANCE) .add(MaterializedViewFilterProjectAggregateRule.INSTANCE) + .add(MaterializedViewFilterScanRule.INSTANCE) + .add(MaterializedViewFilterProjectScanRule.INSTANCE) + .add(MaterializedViewProjectScanRule.INSTANCE) + .add(MaterializedViewProjectFilterScanRule.INSTANCE) .build(); public static final List DPHYP_REORDER_RULES = ImmutableList.builder() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java new file mode 100644 index 00000000000000..7063030f24d70d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * MaterializedViewFilterProjectScanRule + */ +public class MaterializedViewFilterProjectScanRule extends MaterializedViewScanRule { + + public static final MaterializedViewFilterProjectScanRule INSTANCE = new MaterializedViewFilterProjectScanRule(); + + @Override + public List buildRules() { + return ImmutableList.of( + logicalFilter(logicalProject(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> { + LogicalFilter> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java new file mode 100644 index 00000000000000..4cdde78ca4d2f5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * MaterializedViewFilterScanRule + */ +public class MaterializedViewFilterScanRule extends MaterializedViewScanRule { + + public static final MaterializedViewFilterScanRule INSTANCE = new MaterializedViewFilterScanRule(); + + @Override + public List buildRules() { + return ImmutableList.of( + logicalFilter(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> { + LogicalFilter root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_SCAN)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java new file mode 100644 index 00000000000000..55f28b949049e6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * MaterializedViewProjectFilterScanRule + */ +public class MaterializedViewProjectFilterScanRule extends MaterializedViewScanRule { + + public static final MaterializedViewProjectFilterScanRule INSTANCE = new MaterializedViewProjectFilterScanRule(); + + @Override + public List buildRules() { + return ImmutableList.of( + logicalProject(logicalFilter(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> { + LogicalProject> root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java new file mode 100644 index 00000000000000..d73b31f2c7cb49 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * MaterializedViewProjectScanRule + */ +public class MaterializedViewProjectScanRule extends MaterializedViewScanRule { + + public static final MaterializedViewProjectScanRule INSTANCE = new MaterializedViewProjectScanRule(); + + @Override + public List buildRules() { + return ImmutableList.of( + logicalProject(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> { + LogicalProject root = ctx.root; + return rewrite(root, ctx.cascadesContext); + }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_SCAN)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java index c9624c176b9fd9..b3e742c64cba58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java @@ -17,17 +17,67 @@ package org.apache.doris.nereids.rules.exploration.mv; -import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import com.google.common.collect.ImmutableSet; import java.util.List; +import java.util.stream.Collectors; /** * This is responsible for single table rewriting according to different pattern * */ -public class MaterializedViewScanRule extends AbstractMaterializedViewRule { +public abstract class MaterializedViewScanRule extends AbstractMaterializedViewRule { + + @Override + protected Plan rewriteQueryByView(MatchMode matchMode, + StructInfo queryStructInfo, + StructInfo viewStructInfo, + SlotMapping targetToSourceMapping, + Plan tempRewritedPlan, + MaterializationContext materializationContext) { + // Rewrite top projects, represent the query projects by view + List expressionsRewritten = rewriteExpression( + queryStructInfo.getExpressions(), + queryStructInfo.getTopPlan(), + materializationContext.getMvExprToMvScanExprMapping(), + targetToSourceMapping, + true, + queryStructInfo.getTableBitSet() + ); + // Can not rewrite, bail out + if (expressionsRewritten.isEmpty()) { + materializationContext.recordFailReason(queryStructInfo, + "Rewrite expressions by view in scan fail", + () -> String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n" + + "targetToSourceMapping = %s", queryStructInfo.getExpressions(), + materializationContext.getMvExprToMvScanExprMapping(), + targetToSourceMapping)); + return null; + } + return new LogicalProject<>( + expressionsRewritten.stream() + .map(expression -> expression instanceof NamedExpression ? expression : new Alias(expression)) + .map(NamedExpression.class::cast) + .collect(Collectors.toList()), + tempRewritedPlan); + } + /** + * Check scan is whether valid or not. Support join's input only support project, filter, join, + * logical relation, simple aggregate node. Con not have aggregate above on join. + * Join condition should be slot reference equals currently. + */ @Override - public List buildRules() { - return null; + protected boolean checkPattern(StructInfo structInfo) { + PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); + return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) + && !checkContext.isContainsTopAggregate(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 757004071ee4d3..1870be9212bebc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -81,6 +81,7 @@ */ public class StructInfo { public static final PlanPatternChecker PLAN_PATTERN_CHECKER = new PlanPatternChecker(); + public static final ScanPlanPatternChecker SCAN_PLAN_PATTERN_CHECKER = new ScanPlanPatternChecker(); // struct info splitter public static final PlanSplitter PLAN_SPLITTER = new PlanSplitter(); private static final RelationCollector RELATION_COLLECTOR = new RelationCollector(); @@ -606,6 +607,39 @@ private Boolean doVisit(Plan plan, PlanCheckContext checkContext) { } } + /** + * ScanPlanPatternChecker, this is used to check the plan pattern is valid or not + */ + public static class ScanPlanPatternChecker extends DefaultPlanVisitor { + + @Override + public Boolean visitGroupPlan(GroupPlan groupPlan, PlanCheckContext checkContext) { + return groupPlan.getGroup().getLogicalExpressions().stream() + .anyMatch(logicalExpression -> logicalExpression.getPlan().accept(this, checkContext)); + } + + @Override + public Boolean visit(Plan plan, PlanCheckContext checkContext) { + if (plan instanceof Filter + || plan instanceof Project + || plan instanceof CatalogRelation + || plan instanceof GroupPlan) { + return doVisit(plan, checkContext); + } + return false; + } + + private Boolean doVisit(Plan plan, PlanCheckContext checkContext) { + for (Plan child : plan.children()) { + boolean valid = child.accept(this, checkContext); + if (!valid) { + return false; + } + } + return true; + } + } + /** * Add predicates on base table when materialized view scan contains invalid partitions */ diff --git a/regression-test/data/nereids_rules_p0/mv/scan/scan_table.out b/regression-test/data/nereids_rules_p0/mv/scan/scan_table.out new file mode 100644 index 00000000000000..9af8d1bfb63def --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/scan/scan_table.out @@ -0,0 +1,33 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +4 3 3 4 5.50 6.50 7.50 8.50 o k 2023-12-11 2023-12-09 2023-12-10 a b yyyyyyyyy +5 2 3 6 7.50 8.50 9.50 10.50 k o 2023-12-12 2023-12-12 2023-12-13 c d xxxxxxxxx + +-- !query1_0_after -- +4 3 3 4 5.50 6.50 7.50 8.50 o k 2023-12-11 2023-12-09 2023-12-10 a b yyyyyyyyy +5 2 3 6 7.50 8.50 9.50 10.50 k o 2023-12-12 2023-12-12 2023-12-13 c d xxxxxxxxx + +-- !query1_1_before -- +1 2 3 4 5.50 6.50 7.50 8.50 o k 2023-12-08 2023-12-09 2023-12-10 a b yyyyyyyyy +2 4 3 4 5.50 6.50 7.50 8.50 o k 2023-12-09 2023-12-09 2023-12-10 a b yyyyyyyyy +3 2 4 4 5.50 6.50 7.50 8.50 o k 2023-12-10 2023-12-09 2023-12-10 a b yyyyyyyyy +4 3 3 4 5.50 6.50 7.50 8.50 o k 2023-12-11 2023-12-09 2023-12-10 a b yyyyyyyyy + +-- !query1_1_after -- +1 2 3 4 5.50 6.50 7.50 8.50 o k 2023-12-08 2023-12-09 2023-12-10 a b yyyyyyyyy +2 4 3 4 5.50 6.50 7.50 8.50 o k 2023-12-09 2023-12-09 2023-12-10 a b yyyyyyyyy +3 2 4 4 5.50 6.50 7.50 8.50 o k 2023-12-10 2023-12-09 2023-12-10 a b yyyyyyyyy +4 3 3 4 5.50 6.50 7.50 8.50 o k 2023-12-11 2023-12-09 2023-12-10 a b yyyyyyyyy + +-- !query1_2_before -- +6 2023-12-13 + +-- !query1_2_after -- +6 2023-12-13 + +-- !query1_3_before -- +6 + +-- !query1_3_after -- +6 + diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy index 192f991647cb3d..b02df2b99930e7 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy @@ -712,4 +712,176 @@ suite("partition_mv_rewrite_dimension_1") { // } // sql """DROP MATERIALIZED VIEW IF EXISTS ${rewriting_mv_name_1};""" + + def create_mv_lineitem_without_partition = { mv_name, mv_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};""" + sql """DROP TABLE IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + ${mv_sql} + """ + } + + // single table + mv_name_1 = "single_tb_mv_1" + def single_table_mv_stmt_1 = """ + select l_Shipdate, l_partkey, l_suppkey + from lineitem_1 + where l_commitdate like '2023-10-%' + """ + + create_mv_lineitem_without_partition(mv_name_1, single_table_mv_stmt_1) + job_name_1 = getJobName(db, mv_name_1) + waitingMTMVTaskFinished(job_name_1) + + def single_table_query_stmt_1 = """ + select l_Shipdate, l_partkey, l_suppkey + from lineitem_1 + where l_commitdate like '2023-10-%' + """ + def single_table_query_stmt_2 = """ + select l_Shipdate, l_partkey, l_suppkey + from lineitem_1 + where l_commitdate like '2023-10-%' and l_partkey > 0 + 1 + """ + + explain { + sql("${single_table_query_stmt_1}") + contains "${mv_name_1}(${mv_name_1})" + } + compare_res(single_table_query_stmt_1 + " order by 1,2,3") + + explain { + sql("${single_table_query_stmt_2}") + contains "${mv_name_1}(${mv_name_1})" + } + compare_res(single_table_query_stmt_2 + " order by 1,2,3") + + + single_table_mv_stmt_1 = """ + select sum(o_totalprice) as sum_total, + max(o_totalpricE) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders_1 where o_orderdate >= '2022-10-17' + interval '1' year + """ + + def create_mv_orders_without_partition = { mv_name, mv_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};""" + sql """DROP TABLE IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + ${mv_sql} + """ + } + + create_mv_orders_without_partition(mv_name_1, single_table_mv_stmt_1) + job_name_1 = getJobName(db, mv_name_1) + waitingMTMVTaskFinished(job_name_1) + + // not support currently +// single_table_query_stmt_1 = """ +// select sum(o_totalprice) as sum_total, +// max(o_totalpricE) as max_total, +// min(o_totalprice) as min_total, +// count(*) as count_all, +// bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, +// bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 +// from orders_1 where o_orderdate >= '2022-10-17' + interval '1' year +// """ +// single_table_query_stmt_2 = """ +// select sum(o_totalprice) as sum_total, +// max(o_totalpricE) as max_total, +// min(o_totalprice) as min_total, +// count(*) as count_all, +// bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, +// bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 +// from orders_1 where o_orderdate > '2022-10-17' + interval '1' year +// """ +// explain { +// sql("${single_table_query_stmt_1}") +// contains "${mv_name_1}(${mv_name_1})" +// } +// compare_res(single_table_query_stmt_1 + " order by 1,2,3,4") +// explain { +// sql("${single_table_query_stmt_2}") +// contains "${mv_name_1}(${mv_name_1})" +// } +// compare_res(single_table_query_stmt_2 + " order by 1,2,3,4") + + + single_table_mv_stmt_1 = """ + select l_Shipdate, l_partkey, l_suppkey + from lineitem_1 + where l_commitdate in (select l_commitdate from lineitem_1) + """ + + create_mv_lineitem_without_partition(mv_name_1, single_table_mv_stmt_1) + job_name_1 = getJobName(db, mv_name_1) + waitingMTMVTaskFinished(job_name_1) + + single_table_query_stmt_1 = """ + select l_Shipdate, l_partkey, l_suppkey + from lineitem_1 + where l_commitdate in (select l_commitdate from lineitem_1) + """ + explain { + sql("${single_table_query_stmt_1}") + contains "${mv_name_1}(${mv_name_1})" + } + compare_res(single_table_query_stmt_1 + " order by 1,2,3") + +// not supported currently +// single_table_mv_stmt_1 = """ +// select l_Shipdate, l_partkey, l_suppkey +// from lineitem_1 +// where exists (select l_commitdate from lineitem_1 where l_commitdate like "2023-10-17") +// """ +// +// create_mv_lineitem_without_partition(mv_name_1, single_table_mv_stmt_1) +// job_name_1 = getJobName(db, mv_name_1) +// waitingMTMVTaskFinished(job_name_1) +// +// single_table_query_stmt_1 = """ +// select l_Shipdate, l_partkey, l_suppkey +// from lineitem_1 +// where exists (select l_commitdate from lineitem_1 where l_commitdate like "2023-10-17") +// """ +// explain { +// sql("${single_table_query_stmt_1}") +// contains "${mv_name_1}(${mv_name_1})" +// } +// compare_res(single_table_query_stmt_1 + " order by 1,2,3") +// +// +// single_table_mv_stmt_1 = """ +// select t.l_Shipdate, t.l_partkey, t.l_suppkey +// from (select * from lineitem_1) as t +// where exists (select l_commitdate from lineitem_1 where l_commitdate like "2023-10-17") +// """ +// +// create_mv_lineitem_without_partition(mv_name_1, single_table_mv_stmt_1) +// job_name_1 = getJobName(db, mv_name_1) +// waitingMTMVTaskFinished(job_name_1) +// +// single_table_query_stmt_1 = """ +// select t.l_Shipdate, t.l_partkey, t.l_suppkey +// from (select * from lineitem_1) as t +// where exists (select l_commitdate from lineitem_1 where l_commitdate like "2023-10-17") +// """ +// explain { +// sql("${single_table_query_stmt_1}") +// contains "${mv_name_1}(${mv_name_1})" +// } +// compare_res(single_table_query_stmt_1 + " order by 1,2,3") } diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 4a0f513f0fae88..52b58b29be97ff 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -278,7 +278,7 @@ suite("nested_mtmv") { """ explain { sql("${query_stmt_2}") - contains "${mv_level3_name}(${mv_level3_name})" + contains "${mv_level4_name}(${mv_level4_name})" } compare_res(query_stmt_2 + " order by 1,2,3,4,5,6,7") diff --git a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy new file mode 100644 index 00000000000000..b25e497379f567 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("mv_scan_table") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "set runtime_filter_mode=OFF" + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_materialized_view_rewrite=true" + sql "SET enable_nereids_timeout = false" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) (PARTITION `day_2` VALUES LESS THAN ('2023-12-30')) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) (PARTITION `day_1` VALUES LESS THAN ('2023-12-30')) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + + """ + + sql """analyze table orders with sync;""" + sql """analyze table lineitem with sync;""" + sql """analyze table partsupp with sync;""" + + // with filter + def mv1_0 = + """ + select * + from lineitem where l_orderkey > 2 + """ + def query1_0 = """ + select * + from lineitem where l_orderkey > 3 + """ + order_qt_query1_0_before "${query1_0}" + check_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + def mv1_1 = + """ + select * + from lineitem where l_comment like '%yy%' + """ + def query1_1 = """ + select * + from lineitem where l_comment like '%yy%' + """ + order_qt_query1_1_before "${query1_1}" + check_mv_rewrite_success(db, mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + + def mv1_2 = + """ + select * + from lineitem where l_comment like '%xx%' + """ + def query1_2 = """ + select l_linenumber, l_receiptdate + from lineitem where l_comment like '%xx%' + """ + order_qt_query1_2_before "${query1_2}" + check_mv_rewrite_success(db, mv1_2, query1_2, "mv1_2") + order_qt_query1_2_after "${query1_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" + + + def mv1_3 = + """ + select l_linenumber, l_receiptdate + from lineitem where l_comment like '%xx%' + """ + def query1_3 = """ + select l_linenumber + from lineitem where l_comment like '%xx%' + """ + order_qt_query1_3_before "${query1_3}" + check_mv_rewrite_success(db, mv1_3, query1_3, "mv1_3") + order_qt_query1_3_after "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" +}