From f0cb9978cb48a43beeb5762ca5e4d9bd5ef9e956 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:09:18 +0800 Subject: [PATCH 1/7] [test](mv) Insert into more data when first insert into to make sure using sync mv (#43010) Root Cause Analysis: Currently, the statistics reported by BE (Backend) nodes have higher priority than those from ANALYZE statements. During the first INSERT INTO operation, the system waits for row count reports from all tablets before updating the table statistics. Subsequent INSERT INTO operations cannot obtain the status of all tablets, so the system continues to use the statistical information from the first INSERT INTO operation. This leads to a lower estimated cost for the original table's query plan, resulting in the selection of the original table's query plan instead of the materialized view. Conclusion: The test case should be modified to include a larger dataset in the first INSERT INTO operation, which will increase the likelihood of utilizing the materialized view. This is because the cost estimation will better reflect the actual data distribution and size, leading to more accurate plan selection. --- .../data/mv_p0/ssb/q_1_1/q_1_1.out | 4 +++ .../suites/mv_p0/ssb/q_1_1/q_1_1.groovy | 30 +++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out index 1f27d180c736ce..1da845620e8dd7 100644 --- a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out +++ b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out @@ -1,6 +1,10 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container +19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container 19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container -- !select_mv -- diff --git a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy index d3ad529439971b..30959801e72af1 100644 --- a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy @@ -63,14 +63,6 @@ suite ("mv_ssb_q_1_1") { ) ENGINE=OLAP DUPLICATE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`) COMMENT "OLAP" - PARTITION BY RANGE(`LO_ORDERDATE`) - (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 48 PROPERTIES ( "replication_num" = "1", @@ -80,7 +72,13 @@ suite ("mv_ssb_q_1_1") { ); """ - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" + // Add more data when insert into firstly to make sure use mv easier + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" createMV ("""create materialized view lineorder_q_1_1 as SELECT LO_ORDERKEY, SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue @@ -93,20 +91,20 @@ suite ("mv_ssb_q_1_1") { GROUP BY LO_ORDERKEY;""") - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;" - explain { - sql("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue + sql "analyze table lineorder_flat with sync;" + mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE LO_ORDERDATE >= 19930101 AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 - AND LO_QUANTITY < 25;""") - contains "(lineorder_q_1_1)" - } + AND LO_QUANTITY < 25;""", "lineorder_q_1_1") + qt_select_mv """SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE @@ -115,6 +113,7 @@ suite ("mv_ssb_q_1_1") { AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 AND LO_QUANTITY < 25;""" + sql""" drop materialized view lineorder_q_1_1 on lineorder_flat; """ qt_select """SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue @@ -124,4 +123,5 @@ suite ("mv_ssb_q_1_1") { AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 AND LO_QUANTITY < 25;""" + } From ec9a648584d2575e1818fa91e7fe40eda4dd1242 Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 2 Dec 2024 19:14:32 +0800 Subject: [PATCH 2/7] [fix](mtmv) Fix data wrong when query with table operator such as TABLESAMPLE or tablet and so on (#43030) Related PR: #28064 Materialized view is as following: CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH AUTO ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS select * from orders If run query as following, should rewrite fail by materialized view above to make sure data correctness select * from orders TABLET(110); select * from orders index query_index_test; select * from orders TABLESAMPLE(20 percent); select * from orders_partition PARTITION (day_2); At before, this would rewrite by materialized view succesfully and the result data is wrong, This pr fix this. --- .../org/apache/doris/mtmv/MTMVPlanUtil.java | 6 + .../nereids/rules/analysis/BindRelation.java | 4 + .../mv/InitMaterializationContextHook.java | 5 + .../exploration/mv/MaterializedViewUtils.java | 69 ++++++- .../rules/exploration/mv/StructInfo.java | 7 + .../rules/rewrite/PruneEmptyPartition.java | 7 +- .../rules/rewrite/PruneOlapScanTablet.java | 6 +- .../plans/commands/info/CreateMTMVInfo.java | 3 + .../trees/plans/logical/LogicalOlapScan.java | 55 +++-- .../rewrite/PruneOlapScanTabletTest.java | 21 +- .../doris/nereids/sqltest/SqlTestBase.java | 3 + .../doris/regression/suite/Suite.groovy | 16 ++ .../is_in_debug_mode/is_in_debug_mode.groovy | 156 ++++++++++++++ .../with_table_operator.groovy | 195 ++++++++++++++++++ 14 files changed, 531 insertions(+), 22 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy create mode 100644 regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index eb47999117872e..e2c64b5e5b7c94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -60,6 +60,12 @@ public static ConnectContext createMTMVContext(MTMV mtmv) { ctx.setThreadLocalInfo(); ctx.getSessionVariable().enableFallbackToOriginalPlanner = false; ctx.getSessionVariable().enableNereidsDML = true; + // Debug session variable should be disabled when refreshed + ctx.getSessionVariable().skipDeletePredicate = false; + ctx.getSessionVariable().skipDeleteBitmap = false; + ctx.getSessionVariable().skipDeleteSign = false; + ctx.getSessionVariable().skipStorageEngineMerge = false; + ctx.getSessionVariable().showHiddenColumns = false; ctx.getSessionVariable().allowModifyMaterializedViewData = true; // Disable add default limit rule to avoid refresh data wrong ctx.getSessionVariable().setDisableNereidsRules( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index e773d2721a9d6e..36363650cd099e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -237,6 +237,10 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, unboundRelation.getTableSample()); } } + if (!tabletIds.isEmpty()) { + // This tabletIds is set manually, so need to set specifiedTabletIds + scan = scan.withManuallySpecifiedTabletIds(tabletIds); + } if (needGenerateLogicalAggForRandomDistAggTable(scan)) { // it's a random distribution agg table // add agg on olap scan diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 98866691ad87dd..37d66b101cfa4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -69,6 +69,11 @@ public void initMaterializationContext(CascadesContext cascadesContext) { * @param cascadesContext current cascadesContext in the planner */ protected void doInitMaterializationContext(CascadesContext cascadesContext) { + if (cascadesContext.getConnectContext().getSessionVariable().isInDebugMode()) { + LOG.info(String.format("MaterializationContext init return because is in debug mode, current queryId is %s", + cascadesContext.getConnectContext().getQueryIdentifier())); + return; + } // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index e1f9a4a6a6ac3f..8495349c643f54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -25,14 +25,21 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.StructInfoMap; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.analysis.BindRelation; import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.rewrite.EliminateSort; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -47,19 +54,25 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.HashMultimap; @@ -299,7 +312,50 @@ public static List extractNondeterministicFunction(Plan plan) { return nondeterministicFunctions; } - private static final class TableQueryOperatorChecker extends DefaultPlanVisitor { + /** + * createMTMVCache from querySql + */ + public static MTMVCache createMTMVCache(String querySql, ConnectContext connectContext) { + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(querySql); + StatementContext mvSqlStatementContext = new StatementContext(connectContext, + new OriginStatement(querySql, 0)); + NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); + if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { + mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); + } + // Can not convert to table sink, because use the same column from different table when self join + // the out slot is wrong + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainCommand.ExplainLevel.ALL_PLAN); + Plan originPlan = planner.getRewrittenPlan(); + // Eliminate result sink because sink operator is useless in query rewrite by materialized view + // and the top sort can also be removed + Plan mvPlan = originPlan.accept(new DefaultPlanRewriter() { + @Override + public Plan visitLogicalResultSink(LogicalResultSink logicalResultSink, Object context) { + return logicalResultSink.child().accept(this, context); + } + }, null); + // Optimize by rules to remove top sort + CascadesContext parentCascadesContext = CascadesContext.initContext(mvSqlStatementContext, mvPlan, + PhysicalProperties.ANY); + mvPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, childContext -> { + Rewriter.getCteChildrenRewriter(childContext, + ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); + return childContext.getRewritePlan(); + }, mvPlan, originPlan); + return new MTMVCache(mvPlan, originPlan, + planner.getCascadesContext().getMemo().getRoot().getStatistics(), null); + } + + /** + * Check the query if Contains query operator + * Such sql as following should return true + * select * from orders TABLET(10098) because TABLET(10098) should return true + * select * from orders_partition PARTITION (day_2) because PARTITION (day_2) + * select * from orders index query_index_test because index query_index_test + * select * from orders TABLESAMPLE(20 percent) because TABLESAMPLE(20 percent) + * */ + public static final class TableQueryOperatorChecker extends DefaultPlanVisitor { public static final TableQueryOperatorChecker INSTANCE = new TableQueryOperatorChecker(); @Override @@ -310,12 +366,20 @@ public Boolean visitLogicalRelation(LogicalRelation relation, Void context) { if (relation instanceof LogicalOlapScan) { LogicalOlapScan logicalOlapScan = (LogicalOlapScan) relation; if (logicalOlapScan.getTableSample().isPresent()) { + // Contain sample, select * from orders TABLESAMPLE(20 percent) return true; } - if (!logicalOlapScan.getSelectedTabletIds().isEmpty()) { + if (!logicalOlapScan.getManuallySpecifiedTabletIds().isEmpty()) { + // Contain tablets, select * from orders TABLET(10098) because TABLET(10098) return true; } if (!logicalOlapScan.getManuallySpecifiedPartitions().isEmpty()) { + // Contain specified partitions, select * from orders_partition PARTITION (day_2) + return true; + } + if (logicalOlapScan.getSelectedIndexId() != logicalOlapScan.getTable().getBaseIndexId()) { + // Contains select index or use sync mv in rbo rewrite + // select * from orders index query_index_test return true; } } @@ -492,6 +556,7 @@ public Void visitLogicalWindow(LogicalWindow window, IncrementCh @Override public Void visit(Plan plan, IncrementCheckerContext context) { if (plan instanceof LogicalProject + || plan instanceof LogicalLimit || plan instanceof LogicalFilter || plan instanceof LogicalJoin || plan instanceof LogicalAggregate diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index fa29d4d0e123be..e1802981274ac0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -28,6 +28,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.copier.DeepCopierContext; import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; @@ -36,6 +37,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.ObjectId; @@ -326,6 +328,11 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable cascadesContext); valid = valid && hyperGraph.getNodes().stream().allMatch(n -> ((StructInfoNode) n).getExpressions() != null); + // if relationList has any relation which contains table operator, + // such as query with sample, index, table, is invalid + boolean invalid = relationList.stream().anyMatch(relation -> + ((AbstractPlan) relation).accept(TableQueryOperatorChecker.INSTANCE, null)); + valid = valid && !invalid; // collect predicate from top plan which not in hyper graph Set topPlanPredicates = new LinkedHashSet<>(); topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java index 5465211c73e793..c7b8f452afbe66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java @@ -36,11 +36,16 @@ public Rule build() { return logicalOlapScan().thenApply(ctx -> { LogicalOlapScan scan = ctx.root; OlapTable table = scan.getTable(); - List ids = table.selectNonEmptyPartitionIds(scan.getSelectedPartitionIds()); + List partitionIdsToPrune = scan.getSelectedPartitionIds(); + List ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune); if (ids.isEmpty()) { return new LogicalEmptyRelation(ConnectContext.get().getStatementContext().getNextRelationId(), scan.getOutput()); } + if (partitionIdsToPrune.equals(ids)) { + // Not Prune actually, return directly + return null; + } return scan.withSelectedPartitionIds(ids); }).toRule(RuleType.PRUNE_EMPTY_PARTITION); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java index 4b86b7a3706f0e..fc212550b6f364 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java @@ -53,7 +53,7 @@ public Rule build() { LogicalOlapScan olapScan = filter.child(); OlapTable table = olapScan.getTable(); Builder selectedTabletIdsBuilder = ImmutableList.builder(); - if (olapScan.getSelectedTabletIds().isEmpty()) { + if (olapScan.getManuallySpecifiedTabletIds().isEmpty()) { for (Long id : olapScan.getSelectedPartitionIds()) { Partition partition = table.getPartition(id); MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId()); @@ -64,10 +64,10 @@ public Rule build() { partition.getDistributionInfo())); } } else { - selectedTabletIdsBuilder.addAll(olapScan.getSelectedTabletIds()); + selectedTabletIdsBuilder.addAll(olapScan.getManuallySpecifiedTabletIds()); } List selectedTabletIds = selectedTabletIdsBuilder.build(); - if (new HashSet<>(selectedTabletIds).equals(new HashSet<>(olapScan.getSelectedTabletIds()))) { + if (new HashSet<>(selectedTabletIds).equals(new HashSet<>(olapScan.getManuallySpecifiedTabletIds()))) { return null; } return filter.withChildren(olapScan.withSelectedTabletIds(selectedTabletIds)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 837d9478c89327..ed3269d7142bb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -156,6 +156,9 @@ public void analyze(ConnectContext ctx) throws Exception { if (!InternalCatalog.INTERNAL_CATALOG_NAME.equals(mvName.getCtl())) { throw new AnalysisException("Only support creating asynchronous materialized views in internal catalog"); } + if (ctx.getSessionVariable().isInDebugMode()) { + throw new AnalysisException("Create materialized view fail, because is in debug mode"); + } try { FeNameFormat.checkTableName(mvName.getTbl()); } catch (org.apache.doris.common.AnalysisException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 2f0b3b0a769cb1..ca4619dd00dac2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -92,6 +92,12 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan */ private final List selectedTabletIds; + /** + * Selected tablet ids to read data from, this would be set if user query with tablets manually + * Such as select * from orders TABLET(100); + */ + private final List manuallySpecifiedTabletIds; + /////////////////////////////////////////////////////////////////////////// // Members for partition ids. /////////////////////////////////////////////////////////////////////////// @@ -120,12 +126,16 @@ public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } + /** + * LogicalOlapScan construct method + */ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(), - Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of()); + Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(), + ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List tabletIds, @@ -133,7 +143,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, -1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of()); + tableSample, false, ImmutableMap.of(), ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List specifiedPartitions, @@ -142,7 +152,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, -1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of()); + tableSample, false, ImmutableMap.of(), ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List tabletIds, @@ -151,7 +161,8 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L this(id, table, qualifier, Optional.empty(), Optional.empty(), selectedPartitionIds, false, tabletIds, selectedIndexId, true, preAggStatus, - specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of()); + specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of(), + ImmutableList.of()); } /** @@ -164,7 +175,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, PreAggStatus preAggStatus, List specifiedPartitions, List hints, Map, Slot> cacheSlotWithSlotName, Optional tableSample, boolean directMvScan, - Map>> colToSubPathsMap) { + Map>> colToSubPathsMap, List specifiedTabletIds) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); Preconditions.checkArgument(selectedPartitionIds != null, @@ -175,6 +186,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, this.indexSelected = indexSelected; this.preAggStatus = preAggStatus; this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions); + this.manuallySpecifiedTabletIds = ImmutableList.copyOf(specifiedTabletIds); if (selectedPartitionIds.isEmpty()) { this.selectedPartitionIds = ImmutableList.of(); @@ -233,6 +245,7 @@ public boolean equals(Object o) { && partitionPruned == that.partitionPruned && Objects.equals(preAggStatus, that.preAggStatus) && Objects.equals(selectedTabletIds, that.selectedTabletIds) && Objects.equals(manuallySpecifiedPartitions, that.manuallySpecifiedPartitions) + && Objects.equals(manuallySpecifiedTabletIds, that.manuallySpecifiedTabletIds) && Objects.equals(selectedPartitionIds, that.selectedPartitionIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample, that.tableSample); @@ -241,8 +254,8 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash(super.hashCode(), selectedIndexId, indexSelected, preAggStatus, cacheSlotWithSlotName, - selectedTabletIds, partitionPruned, manuallySpecifiedPartitions, selectedPartitionIds, hints, - tableSample); + selectedTabletIds, partitionPruned, manuallySpecifiedTabletIds, manuallySpecifiedPartitions, + selectedPartitionIds, hints, tableSample); } @Override @@ -251,7 +264,7 @@ public LogicalOlapScan withGroupExpression(Optional groupExpres groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } @Override @@ -260,7 +273,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr return new LogicalOlapScan(relationId, (Table) table, qualifier, groupExpression, logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { @@ -268,7 +281,7 @@ public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withMaterializedIndexSelected(long indexId) { @@ -276,7 +289,7 @@ public LogicalOlapScan withMaterializedIndexSelected(long indexId) { Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, - tableSample, directMvScan, colToSubPathsMap); + tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { @@ -284,7 +297,7 @@ public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { @@ -292,7 +305,7 @@ public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } public LogicalOlapScan withColToSubPathsMap(Map>> colToSubPathsMap) { @@ -300,7 +313,15 @@ public LogicalOlapScan withColToSubPathsMap(Map>> colTo Optional.empty(), Optional.empty(), selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap); + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); + } + + public LogicalOlapScan withManuallySpecifiedTabletIds(List manuallySpecifiedTabletIds) { + return new LogicalOlapScan(relationId, (Table) table, qualifier, + Optional.empty(), Optional.of(getLogicalProperties()), + selectedPartitionIds, partitionPruned, selectedTabletIds, + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds); } @Override @@ -310,7 +331,7 @@ public LogicalOlapScan withRelationId(RelationId relationId) { Optional.empty(), Optional.empty(), selectedPartitionIds, false, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap); + hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds); } @Override @@ -326,6 +347,10 @@ public List getSelectedTabletIds() { return selectedTabletIds; } + public List getManuallySpecifiedTabletIds() { + return manuallySpecifiedTabletIds; + } + @Override public long getSelectedIndexId() { return selectedIndexId; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java index e7940cdfb9b519..385657cd3f229e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; import org.apache.doris.nereids.trees.expressions.InPredicate; @@ -51,8 +52,9 @@ import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Objects; -class PruneOlapScanTabletTest implements MemoPatternMatchSupported { +class PruneOlapScanTabletTest extends SqlTestBase implements MemoPatternMatchSupported { @Test void testPruneOlapScanTablet(@Mocked OlapTable olapTable, @@ -154,4 +156,21 @@ void testPruneOlapScanTablet(@Mocked OlapTable olapTable, ) ); } + + @Test + void testPruneOlapScanTabletWithManually() { + String sql = "select * from T4 TABLET(110) where id > 8"; + PlanChecker.from(connectContext) + .analyze(sql) + .applyTopDown(new PruneOlapScanTablet()) + .matches( + logicalFilter( + logicalOlapScan().when(s -> + Objects.equals(s.getSelectedTabletIds(), Lists.newArrayList(110L)) + && Objects.equals(s.getManuallySpecifiedTabletIds(), + Lists.newArrayList(110L)) + ) + ) + ); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java index f5b301dcc49b25..9ebbc22a85c826 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java @@ -87,6 +87,9 @@ protected void runBeforeAll() throws Exception { + " score bigint\n" + ")\n" + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`id`)\n" + + "(\n" + + ")\n" + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index fa43c3852778f2..06630e89cfd35c 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1515,6 +1515,22 @@ class Suite implements GroovyInterceptable { sql "analyze table ${db}.${mv_name} with sync;" } + def create_async_partition_mv = { db, mv_name, mv_sql, partition_col -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${db}.${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${db}.${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY ${partition_col} + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + sql "analyze table ${db}.${mv_name} with sync;" + } + // mv not part in rewrite process def mv_not_part_in = { query_sql, mv_name -> logger.info("query_sql = " + query_sql + ", mv_names = " + mv_name) diff --git a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy new file mode 100644 index 00000000000000..15d93e32f65dc2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("is_in_debug_mode") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + UNIQUE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + create_async_mv(db, "basic_mv", """ + select * from orders where o_orderkey > 1; + """) + + sql """set skip_delete_sign = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_delete_sign = false;""" + + + sql """set skip_storage_engine_merge = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_storage_engine_merge = false;""" + + + sql """set skip_delete_bitmap = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_delete_bitmap = false;""" + + + sql """set skip_delete_predicate = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set skip_delete_predicate = false;""" + + + sql """set show_hidden_columns = true;""" + mv_not_part_in("""select * from orders where o_orderkey > 1;""", "basic_mv") + try { + sql """ + CREATE MATERIALIZED VIEW test_create_mv + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS select * from orders where o_orderkey > 2; + """ + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("because is in debug mode")) + } + sql """set show_hidden_columns = false;""" + + sql """drop materialized view if exists basic_mv""" +} + diff --git a/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy new file mode 100644 index 00000000000000..5ba2c05053e2a2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/with_table_operator/with_table_operator.groovy @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("with_table_operator") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + drop table if exists orders_partition; + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders_partition ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders_partition values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + create_async_mv(db, "complete_agg_mv", """ + select count(*),o_orderdate + from orders + group by o_orderdate; + """) + + create_async_mv(db, "complete_mv", """ + select * from orders; + """) + + create_async_partition_mv(db, "partition_agg_mv", """ + select count(*),o_orderdate + from orders_partition + group by o_orderdate; + """, "(o_orderdate)") + + create_async_partition_mv(db, "partition_mv", """ + select * from orders_partition; + """, "(o_orderdate)") + + // query rewrite with tablet should fail + mv_rewrite_all_fail(""" + select count(*),o_orderdate + from orders TABLET(110) + group by o_orderdate; + """, ["complete_mv", "complete_agg_mv"]) + + mv_rewrite_all_fail("""select * from orders TABLET(110);""", ["complete_mv", "complete_agg_mv"]) + + // query rewrite with index should fail + createMV(""" + create materialized view query_index_test + as + select o_orderdate, count(*) + from orders + group by o_orderdate; + """) + mv_rewrite_all_fail(""" + select * from orders index query_index_test; + """, ["complete_mv", "complete_agg_mv"]) + + // query rewrite with sample should fail + mv_rewrite_all_fail(""" + select count(*),o_orderdate + from orders TABLESAMPLE(20 percent) + group by o_orderdate; + """, ["complete_mv", "complete_agg_mv"]) + + mv_rewrite_all_fail("""select * from orders TABLESAMPLE(20 percent);""", ["complete_mv", "complete_agg_mv"]) + + // query rewrite with partition should fail + mv_rewrite_all_fail(""" + select count(*),o_orderdate + from orders_partition PARTITION (day_2) + group by o_orderdate; + """, ["partition_mv", "partition_agg_mv"]) + + mv_rewrite_all_fail("""select * from orders_partition PARTITION (day_2);""", ["partition_mv", "partition_agg_mv"]) + + + sql """ + DROP MATERIALIZED VIEW IF EXISTS complete_agg_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS partition_agg_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS complete_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS partition_mv; + """ + sql """ + DROP MATERIALIZED VIEW IF EXISTS query_index_test on orders; + """ +} + From f59403bf72abfa79c143867793d7c5067baff2bb Mon Sep 17 00:00:00 2001 From: seawinde Date: Fri, 29 Nov 2024 14:42:38 +0800 Subject: [PATCH 3/7] [test](mtmv) Add inject statistic when mv rewrite regression test to make sure rewrite result stable (#43785) The result of successful rewriting by the cbo optimizer depends on the statistics. The priority of the optimizer consumption statistics in descending order is 1. the injected statistics 2. the statistics reported by be 3. and the statistics analyzed actively. When the pipeline runs the case, the statistics reported by be may not be timely. Therefore, the outcome that leads to the cbo optimizer's successful selection of overwrites is not very certain, so the statistics are currently injected manually in the test cases --- .../data/mv_p0/ssb/q_1_1/q_1_1.out | 7 +- .../doris/regression/suite/Suite.groovy | 3 +- .../agg_have_dup_base.groovy | 2 + .../agg_state/test_agg_state_max_by.groovy | 3 + .../mv_p0/case_ignore/case_ignore.groovy | 3 + .../suites/mv_p0/count_star/count_star.groovy | 2 + .../suites/mv_p0/dis_26495/dis_26495.groovy | 2 + .../suites/mv_p0/k1ap2spa/k1ap2spa.groovy | 2 + .../suites/mv_p0/k1s2m3/k1s2m3.groovy | 1 + .../k1s2m3_auto_inc/k1s2m3_auto_inc.groovy | 1 + .../multi_agg_with_same_slot.groovy | 1 + .../multi_slot_k123p/multi_slot_k123p.groovy | 1 + .../multi_slot_k1a2p2ap3p.groovy | 1 + .../multi_slot_k1a2p2ap3ps.groovy | 1 + .../multi_slot_k1p2ap3p.groovy | 1 + .../multi_slot_k1p2ap3ps.groovy | 1 + .../mv_p0/mv_with_view/mv_with_view.groovy | 26 +-- .../mv_p0/null_insert/null_insert.groovy | 1 + .../routine_load_hll/routine_load_hll.groovy | 1 + .../multiple_ssb_between.groovy | 51 ++--- .../suites/mv_p0/ssb/q_1_1/q_1_1.groovy | 20 +- .../suites/mv_p0/ssb/q_2_1/q_2_1.groovy | 2 + .../suites/mv_p0/ssb/q_3_1/q_3_1.groovy | 27 +-- .../suites/mv_p0/ssb/q_4_1/q_4_1.groovy | 2 + .../suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy | 1 + .../suites/mv_p0/sum_count/sum_count.groovy | 1 + .../sum_divede_count/sum_devide_count.groovy | 2 + .../suites/mv_p0/test_28741/test_28741.groovy | 2 +- .../test_approx_count_distinct.groovy | 1 + .../suites/mv_p0/test_base/test_base.groovy | 1 + .../mv_p0/test_casewhen/test_casewhen.groovy | 1 + .../test_create_mv/test_create_mv.groovy | 2 + .../test_create_mv_complex_type.groovy | 2 + .../mv_p0/test_doc_e4/test_doc_e4.groovy | 1 + .../test_dup_group_by_mv_abs.groovy | 1 + .../test_dup_group_by_mv_plus.groovy | 1 + .../test_dup_mv_abs/test_dup_mv_abs.groovy | 1 + .../test_dup_mv_bin/test_dup_mv_bin.groovy | 1 + .../test_dup_mv_bitmap_hash.groovy | 3 + .../test_dup_mv_plus/test_dup_mv_plus.groovy | 2 +- .../test_dup_mv_repeat.groovy | 1 + .../test_dup_mv_year/test_dup_mv_year.groovy | 1 + .../suites/mv_p0/test_mv_dp/test_mv_dp.groovy | 2 + .../mv_p0/test_mv_mor/test_mv_mor.groovy | 2 + .../suites/mv_p0/test_ndv/test_ndv.groovy | 1 + .../suites/mv_p0/test_nvl/test_nvl.groovy | 1 + .../suites/mv_p0/test_o2/test_o2.groovy | 1 + .../mv_p0/test_substr/test_substr.groovy | 1 + .../mv_p0/test_tbl_name/test_tbl_name.groovy | 1 + .../test_upper_alias/test_upper_alias.groovy | 1 + .../test_user_activity.groovy | 1 + .../suites/mv_p0/unique/unique.groovy | 1 + .../testAggQueryOnAggMV1.groovy | 1 + .../testAggQueryOnAggMV10.groovy | 1 + .../testAggQueryOnAggMV11.groovy | 1 + .../testAggQueryOnAggMV2.groovy | 1 + .../testAggQueryOnAggMV3.groovy | 2 +- .../testAggQuqeryOnAggMV5.groovy | 1 + .../testAggQuqeryOnAggMV6.groovy | 1 + .../testAggQuqeryOnAggMV7.groovy | 2 + ...testAggregateMVCalcAggFunctionQuery.groovy | 1 + .../testBitmapUnionInQuery.groovy | 2 +- .../testCountDistinctToBitmap.groovy | 4 + .../testIncorrectMVRewriteInSubquery.groovy | 1 + .../testIncorrectRewriteCountDistinct.groovy | 1 + .../testJoinOnLeftProjectToJoin.groovy | 3 + .../mv_p0/ut/testNDVToHll/testNDVToHll.groovy | 1 + .../testOrderByQueryOnProjectView.groovy | 1 + .../testProjectionMV1.groovy | 1 + .../testProjectionMV2.groovy | 1 + .../testProjectionMV3.groovy | 1 + .../testProjectionMV4.groovy | 1 + .../ut/testQueryOnStar/testQueryOnStar.groovy | 1 + .../testSelectMVWithTableAlias.groovy | 2 + .../testSingleMVMultiUsage.groovy | 1 + .../mv_p0/ut/testSubQuery/testSubQuery.groovy | 2 +- .../testUnionDistinct.groovy | 2 + .../mv/agg_on_none_agg/agg_on_none_agg.groovy | 6 + .../mv/agg_variety/agg_variety.groovy | 6 + .../aggregate_with_roll_up.groovy | 3 + .../aggregate_without_roll_up.groovy | 6 + .../mv/availability/grace_period.groovy | 2 + .../materialized_view_switch.groovy | 22 +- .../mv/dimension/dimension_1.groovy | 3 + .../mv/dimension/dimension_2_3.groovy | 3 + .../mv/dimension/dimension_2_4.groovy | 4 + .../mv/dimension/dimension_2_5.groovy | 4 + .../mv/dimension/dimension_2_6.groovy | 6 + .../mv/dimension/dimension_2_full_join.groovy | 2 + .../dimension/dimension_2_inner_join.groovy | 3 + .../dimension_2_left_anti_join.groovy | 3 + .../mv/dimension/dimension_2_left_join.groovy | 3 + .../dimension_2_left_semi_join.groovy | 3 + .../dimension_2_right_anti_join.groovy | 3 + .../dimension/dimension_2_right_join.groovy | 3 + .../dimension_2_right_semi_join.groovy | 3 + .../mv/dimension/dimension_self_conn.groovy | 3 + .../dimension_2_join_agg.groovy | 15 +- .../dimension_join_agg_negative.groovy | 3 + .../filter_equal_or_notequal.groovy | 3 + .../mv/direct_query/direct_query.groovy | 3 + .../mv/grouping_sets/grouping_sets.groovy | 3 + .../join/dphyp_inner/inner_join_dphyp.groovy | 4 + .../join/dphyp_outer/outer_join_dphyp.groovy | 4 + .../mv/join/inner/inner_join.groovy | 4 + .../mv/join/left_outer/outer_join.groovy | 11 + .../inner_join_infer_and_derive.groovy | 3 + .../left_join_infer_and_derive.groovy | 2 + .../right_join_infer_and_derive.groovy | 3 + .../mv/negative/negative_test.groovy | 3 + .../mv/nested/nested_materialized_view.groovy | 4 + .../mv/nested_mtmv/nested_mtmv.groovy | 213 +++++++----------- .../nested_mtmv_rewrite_switch.groovy | 3 + .../mv/partition_mv_rewrite.groovy | 4 + .../mv/scan/scan_table.groovy | 4 + .../union_all_compensate.groovy | 3 + .../partition_curd_union_rewrite.groovy | 3 + .../usercase_union_rewrite.groovy | 3 + .../unsafe_equals/null_un_safe_equals.groovy | 2 + .../mv/variant/variant_mv.groovy | 3 + .../with_auth/with_select_table_auth.groovy | 3 + .../query_with_sql_limit.groovy | 4 + .../mv/newMv/aggHaveDupBase.groovy | 1 + .../mv/newMv/case_ignore.groovy | 2 + .../mv/newMv/dup_gb_mv_abs.groovy | 1 + .../mv/newMv/dup_gb_mv_plus.groovy | 1 + .../mv/newMv/dup_mv_abs.groovy | 2 +- .../mv/newMv/dup_mv_bin.groovy | 1 + .../mv/newMv/dup_mv_bm_hash.groovy | 1 + .../mv/newMv/dup_mv_plus.groovy | 2 +- .../mv/newMv/dup_mv_year.groovy | 1 + .../mv/newMv/multi_slot1.groovy | 1 + .../mv/newMv/multi_slot2.groovy | 1 + .../mv/newMv/multi_slot3.groovy | 1 + .../mv/newMv/multi_slot4.groovy | 1 + .../mv/newMv/multi_slot5.groovy | 1 + .../mv/newMv/multi_slot6.groovy | 1 + .../mv/newMv/single_slot.groovy | 2 +- .../mv/newMv/sum_devide_count.groovy | 2 +- .../mv/newMv/unique_mv.groovy | 1 + .../mv/ut/MVMultiUsage.groovy | 3 + .../nereids_syntax_p0/mv/ut/MVWithAs.groovy | 3 + .../mv/ut/aggCDInBitmap.groovy | 9 + .../mv/ut/aggMVCalcAggFun.groovy | 2 +- .../mv/ut/aggOnAggMV1.groovy | 1 + .../mv/ut/aggOnAggMV10.groovy | 1 + .../mv/ut/aggOnAggMV11.groovy | 3 + .../mv/ut/aggOnAggMV2.groovy | 1 + .../mv/ut/aggOnAggMV3.groovy | 1 + .../mv/ut/aggOnAggMV5.groovy | 2 + .../mv/ut/aggOnAggMV6.groovy | 1 + .../mv/ut/aggOnAggMV7.groovy | 1 + .../mv/ut/bitmapUnionIn.groovy | 3 + .../mv/ut/incMVReInSub.groovy | 3 + .../mv/ut/incRewriteCD.groovy | 3 + .../mv/ut/joinOnLeftPToJoin.groovy | 5 + .../mv/ut/orderByOnPView.groovy | 2 + .../nereids_syntax_p0/mv/ut/projectMV1.groovy | 3 + .../nereids_syntax_p0/mv/ut/projectMV2.groovy | 3 + .../nereids_syntax_p0/mv/ut/projectMV3.groovy | 3 + .../nereids_syntax_p0/mv/ut/projectMV4.groovy | 3 + .../nereids_syntax_p0/mv/ut/subQuery.groovy | 2 +- .../nereids_syntax_p0/mv/ut/unionDis.groovy | 2 + .../nereids_syntax_p0/rollup/agg.groovy | 1 + .../nereids_syntax_p0/rollup/agg_date.groovy | 1 + .../nereids_syntax_p0/rollup/bitmap.groovy | 1 + .../nereids_syntax_p0/rollup/date.groovy | 1 + .../nereids_syntax_p0/rollup/hll/hll.groovy | 1 + .../hll_with_light_sc.groovy | 1 + 169 files changed, 497 insertions(+), 228 deletions(-) diff --git a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out index 1da845620e8dd7..c00bbc9377409e 100644 --- a/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out +++ b/regression-test/data/mv_p0/ssb/q_1_1/q_1_1.out @@ -6,10 +6,13 @@ 19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container 19930101 1 1 1 1 1 1 1 1 1 1 100 1 1 1 2023-06-09 shipmode name address city nation AMERICA phone mktsegment name address city nation AMERICA phone name MFGR#1 category brand color type 4 container 19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container +19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container +19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container +19930101 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2023-06-09 shipmode name address city nation region phone mktsegment name address city nation region phone name mfgr category brand color type 4 container -- !select_mv -- -4 +16 -- !select -- -4 +16 diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 06630e89cfd35c..fe355e01f7753e 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1617,9 +1617,8 @@ class Suite implements GroovyInterceptable { check { result -> boolean success = true; for (String mv_name : mv_names) { - success = success && result.contains("${mv_name} chose") + Assert.assertEquals(true, result.contains("${mv_name} chose")) } - Assert.assertEquals(true, success) } } } diff --git a/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy b/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy index d9f73bcf89f567..d354b4b0e17ff5 100644 --- a/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy +++ b/regression-test/suites/mv_p0/agg_have_dup_base/agg_have_dup_base.groovy @@ -56,6 +56,8 @@ suite ("agg_have_dup_base") { qt_select_mv "select unix_timestamp(k1) tmp,sum(k2) from d_table group by tmp order by tmp;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='5');""" + mv_rewrite_success("select k1,sum(k2),max(k2) from d_table group by k1;", "k12s3m") mv_rewrite_success("select k1,sum(k2) from d_table group by k1;", "k12s3m") diff --git a/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy b/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy index 1e34c70ccec900..29190396230dce 100644 --- a/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy +++ b/regression-test/suites/mv_p0/agg_state/test_agg_state_max_by.groovy @@ -69,6 +69,7 @@ suite ("test_agg_state_max_by") { qt_select_star "select * from d_table order by 1,2;" mv_rewrite_success("select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;", "k1mb") sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;", "k1mb") qt_select_mv "select k1,max_by(k2,k3) from d_table group by k1 order by 1,2;" @@ -101,6 +102,7 @@ suite ("test_agg_state_max_by") { qt_select_star "select * from d_table order by 1,2;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" sql "analyze table d_table with sync;" sql """set enable_stats=false;""" @@ -114,6 +116,7 @@ suite ("test_agg_state_max_by") { qt_select_mv "select k1,max_by(k2,abs(k3)) from d_table group by k1 order by 1,2;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,max_by(k2+k3,abs(k3)) from d_table group by k1 order by 1,2;", "k1mbcp1") mv_rewrite_success("select k1,max_by(k2+k3,k3) from d_table group by k1 order by 1,2;", "k1mbcp2") mv_rewrite_success("select k1,max_by(k2,abs(k3)) from d_table group by k1 order by 1,2;", "k1mbcp3") diff --git a/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy b/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy index 1134d930185b9a..271eb0cb5ed190 100644 --- a/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy +++ b/regression-test/suites/mv_p0/case_ignore/case_ignore.groovy @@ -37,6 +37,8 @@ suite ("case_ignore") { sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" + sql """alter table d_table modify column k4 set stats ('row_count'='4');""" + createMV ("create materialized view k12a as select K1,abs(K2) from d_table;") sql "insert into d_table select -4,-4,-4,'d';" @@ -53,6 +55,7 @@ suite ("case_ignore") { qt_select_mv "select K1,abs(K2) from d_table order by K1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,abs(k2) from d_table order by k1;", "k12a") mv_rewrite_success("select K1,abs(K2) from d_table order by K1;", "k12a") diff --git a/regression-test/suites/mv_p0/count_star/count_star.groovy b/regression-test/suites/mv_p0/count_star/count_star.groovy index 52edda58276ef6..4d951d76fa98a9 100644 --- a/regression-test/suites/mv_p0/count_star/count_star.groovy +++ b/regression-test/suites/mv_p0/count_star/count_star.groovy @@ -64,6 +64,8 @@ suite ("count_star") { qt_select_mv "select count(*) from d_table where k3=1;" sql """set enable_stats=true;""" + + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" mv_rewrite_success("select k1,k4,count(*) from d_table group by k1,k4;", "kstar") mv_rewrite_success("select k1,k4,count(*) from d_table where k1=1 group by k1,k4;", "kstar") mv_rewrite_fail("select k1,k4,count(*) from d_table where k3=1 group by k1,k4;", "kstar") diff --git a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy index e30af70b0041b3..14ebb67ddc8168 100644 --- a/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy +++ b/regression-test/suites/mv_p0/dis_26495/dis_26495.groovy @@ -28,6 +28,8 @@ suite ("dis_26495") { sql """insert into doris_test values (1,2,max_by_state(1,2));""" + sql """alter table doris_test modify column agg_st_1 set stats ('row_count'='1');""" + streamLoad { table "doris_test" set 'column_separator', ',' diff --git a/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy b/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy index cfe2a5307d7ed4..b4c2921669663a 100644 --- a/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy +++ b/regression-test/suites/mv_p0/k1ap2spa/k1ap2spa.groovy @@ -52,6 +52,8 @@ suite ("k1ap2spa") { sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" + mv_rewrite_success("select abs(k1)+1 t,sum(abs(k2+1)) from d_table group by t order by t;", "k1ap2spa") } diff --git a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy index 02617d68f6020b..6c257755596d17 100644 --- a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy +++ b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy @@ -36,6 +36,7 @@ suite ("k1s2m3") { sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" + sql """alter table d_table modify column k1 set stats ('row_count'='6');""" createMV("create materialized view k1s2m3 as select k1,sum(k2*k3) from d_table group by k1;") sql "insert into d_table select -4,-4,-4,'d';" diff --git a/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy b/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy index 367cd9863c1e3e..17583741e68b91 100644 --- a/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy +++ b/regression-test/suites/mv_p0/k1s2m3_auto_inc/k1s2m3_auto_inc.groovy @@ -57,5 +57,6 @@ suite ("k1s2m3_auto_inc") { qt_select_mv "select k3,sum(abs(k2+1)) from d_table group by k3 order by 1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='2');""" mv_rewrite_success("select k3,sum(abs(k2+1)) from d_table group by k3 order by 1;", "k3ap2spa") } diff --git a/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy b/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy index a1d254b333e211..538d050fec8a9d 100644 --- a/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy +++ b/regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy @@ -64,6 +64,7 @@ suite ("multi_agg_with_same_slot") { sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k2,avg(k3),max(k3) from d_table group by k1,k2 order by 1,2;", "kmv") mv_rewrite_success("select k1,k2,avg(k3)+max(k3) from d_table group by k1,k2 order by 1,2;", "kmv") mv_rewrite_success("select k1,k2,avg(k3)+max(k3) from d_table group by grouping sets((k1),(k1,k2),()) order by 1,2;", "kmv") diff --git a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy index 761257230e245d..d06645e00cdfe0 100644 --- a/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k123p/multi_slot_k123p.groovy @@ -66,5 +66,6 @@ suite ("multi_slot_k123p") { qt_select_mv "select k1,version() from d_table order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k2+k3 from d_table order by k1;", "k123p") } diff --git a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy index e64adbb06b2988..fc722c5a07fbe7 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3p/multi_slot_k1a2p2ap3p.groovy @@ -51,6 +51,7 @@ suite ("multi_slot_k1a2p2ap3p") { qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='7');""" mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from d_table order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p") } diff --git a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy index 3a2f58f38d65a9..b2292084504199 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1a2p2ap3ps/multi_slot_k1a2p2ap3ps.groovy @@ -61,6 +61,7 @@ suite ("multi_slot_k1a2p2ap3ps") { qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2 order by abs(k1)+k2;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by abs(k1)+k2+1", "k1a2p2ap3ps") mv_rewrite_fail("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2 order by abs(k1)+k2", "k1a2p2ap3ps") diff --git a/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy b/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy index fa67fd3889d93e..75bbd3fb4b7bb8 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1p2ap3p/multi_slot_k1p2ap3p.groovy @@ -50,5 +50,6 @@ suite ("multi_slot_k1p2ap3p") { qt_select_mv "select k1+1,abs(k2+2)+k3+3 from d_table order by k1+1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1+1,abs(k2+2)+k3+3 from d_table order by k1+1;", "k1p2ap3p") } diff --git a/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy b/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy index e2f3564e2de2cb..4ef8a746a2b797 100644 --- a/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy +++ b/regression-test/suites/mv_p0/multi_slot_k1p2ap3ps/multi_slot_k1p2ap3ps.groovy @@ -49,5 +49,6 @@ suite ("multi_slot_k1p2ap3ps") { qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from d_table group by k1+1 order by k1+1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1+1,sum(abs(k2+2)+k3+3) from d_table group by k1+1 order by k1+1;", "k1p2ap3ps") } diff --git a/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy b/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy index 016dc03753bd9e..69f80e4c96af9e 100644 --- a/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy +++ b/regression-test/suites/mv_p0/mv_with_view/mv_with_view.groovy @@ -36,28 +36,25 @@ suite ("mv_with_view") { sql """insert into d_table select 1,1,1,'a';""" sql """insert into d_table select 2,2,2,'b';""" - createMV("create materialized view k132 as select k1,k3,k2 from d_table;") + createMV("create materialized view k312 as select k3,k1,k2 from d_table;") sql """insert into d_table select 3,-3,null,'c';""" - explain { - sql("select * from d_table order by k1;") - contains "(d_table)" - } + sql "analyze table d_table with sync;" + sql """set enable_stats=false;""" + + mv_rewrite_fail("select * from d_table order by k1;", "k312") qt_select_star "select * from d_table order by k1;" sql """ - drop view if exists v_k132; + drop view if exists v_k312; """ sql """ - create view v_k132 as select k1,k3,k2 from d_table where k1 = 1; + create view v_k312 as select k1,k3,k2 from d_table where k3 = 1; """ - explain { - sql("select * from v_k132 order by k1;") - contains "(k132)" - } - qt_select_mv "select * from v_k132 order by k1;" + mv_rewrite_success("select * from v_k312 order by k1;", "k312") + qt_select_mv "select * from v_k312 order by k1;" sql """ drop view if exists v_k124; @@ -66,9 +63,6 @@ suite ("mv_with_view") { sql """ create view v_k124 as select k1,k2,k4 from d_table where k1 = 1; """ - explain { - sql("select * from v_k124 order by k1;") - contains "(d_table)" - } + mv_rewrite_fail("select * from v_k124 order by k1;", "k312") qt_select_mv "select * from v_k124 order by k1;" } diff --git a/regression-test/suites/mv_p0/null_insert/null_insert.groovy b/regression-test/suites/mv_p0/null_insert/null_insert.groovy index 015406d80a3796..7b6ea058f2d90b 100644 --- a/regression-test/suites/mv_p0/null_insert/null_insert.groovy +++ b/regression-test/suites/mv_p0/null_insert/null_insert.groovy @@ -76,6 +76,7 @@ suite ("null_insert") { GROUP BY date,vid,os,ver,ip_country;""" sql """set enable_stats=true;""" + sql """alter table test modify column date set stats ('row_count'='3');""" mv_rewrite_success("""SELECT date, vid, os, ver, ip_country, hll_union(hll_hash(uid)) FROM test GROUP BY date,vid,os,ver,ip_country;""", "mv_test") diff --git a/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy b/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy index a872e23efc4cce..d34024ebdf5068 100644 --- a/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy +++ b/regression-test/suites/mv_p0/routine_load_hll/routine_load_hll.groovy @@ -58,5 +58,6 @@ suite ("routine_load_hll") { qt_select_mv "select time_stamp, hll_union_agg(device_id) from test group by time_stamp order by 1;" sql """set enable_stats=true;""" + sql """alter table test modify column event_id set stats ('row_count'='2');""" mv_rewrite_success("select time_stamp, hll_union_agg(device_id) from test group by time_stamp order by 1;", "m_view") } diff --git a/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy b/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy index db95a084a69778..a83c78c25225c7 100644 --- a/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy +++ b/regression-test/suites/mv_p0/ssb/multiple_ssb_between/multiple_ssb_between.groovy @@ -65,14 +65,6 @@ suite ("multiple_ssb_between") { ) ENGINE=OLAP DUPLICATE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`) COMMENT "OLAP" - PARTITION BY RANGE(`LO_ORDERDATE`) - (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 48 PROPERTIES ( "replication_num" = "1", @@ -153,17 +145,22 @@ suite ("multiple_ssb_between") { sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (1 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 1 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;" + + sql "analyze table lineorder_flat with sync;" + sql """set enable_stats=true;""" + + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');""" - explain { - sql("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue + mv_rewrite_success("""SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE LO_ORDERDATE >= 19930101 AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT BETWEEN 1 AND 3 - AND LO_QUANTITY < 25;""") - contains "(lineorder_q_1_1)" - } + AND LO_QUANTITY < 25;""", + "lineorder_q_1_1" + ) + qt_select_q_1_1 """SELECT SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue FROM lineorder_flat WHERE @@ -172,16 +169,16 @@ suite ("multiple_ssb_between") { AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25;""" - explain { - sql("""SELECT + mv_rewrite_success("""SELECT SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR, P_BRAND FROM lineorder_flat WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' GROUP BY (LO_ORDERDATE DIV 10000), P_BRAND - ORDER BY YEAR, P_BRAND;""") - contains "(lineorder_q_2_1)" - } + ORDER BY YEAR, P_BRAND;""", + "lineorder_q_2_1" + ) + qt_select_q_2_1 """SELECT SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR, P_BRAND @@ -190,8 +187,7 @@ suite ("multiple_ssb_between") { GROUP BY YEAR, P_BRAND ORDER BY YEAR, P_BRAND;""" - explain { - sql("""SELECT + mv_rewrite_success("""SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, SUM(LO_REVENUE) AS revenue @@ -202,9 +198,9 @@ suite ("multiple_ssb_between") { AND LO_ORDERDATE >= 19920101 AND LO_ORDERDATE <= 19971231 GROUP BY C_NATION, S_NATION, YEAR - ORDER BY YEAR ASC, revenue DESC;""") - contains "(lineorder_q_3_1)" - } + ORDER BY YEAR ASC, revenue DESC;""", + "lineorder_q_3_1") + qt_select_q_3_1 """SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, @@ -218,8 +214,7 @@ suite ("multiple_ssb_between") { GROUP BY C_NATION, S_NATION, YEAR ORDER BY YEAR ASC, revenue DESC;""" - explain { - sql("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, + mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit FROM lineorder_flat @@ -228,9 +223,9 @@ suite ("multiple_ssb_between") { AND S_REGION = 'AMERICA' AND P_MFGR IN ('MFGR#1', 'MFGR#2') GROUP BY YEAR, C_NATION - ORDER BY YEAR ASC, C_NATION ASC;""") - contains "(lineorder_q_4_1)" - } + ORDER BY YEAR ASC, C_NATION ASC;""", + "lineorder_q_4_1") + qt_select_q_4_1 """SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit diff --git a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy index 30959801e72af1..c7aa6e67c6ee8e 100644 --- a/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_1_1/q_1_1.groovy @@ -75,10 +75,12 @@ suite ("mv_ssb_q_1_1") { // Add more data when insert into firstly to make sure use mv easier sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), - (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), - (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), - (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), - (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container'), + (19930101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'shipmode' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' , 'phone' , 'mktsegment' , 'name' , 'address' , 'city' , 'nation' , 'AMERICA' ,'phone', 'name', 'MFGR#1', 'category', 'brand', 'color', 'type', 4 ,'container');""" + + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='6');""" createMV ("""create materialized view lineorder_q_1_1 as SELECT LO_ORDERKEY, SUM(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue @@ -88,12 +90,14 @@ suite ("mv_ssb_q_1_1") { AND LO_ORDERDATE <= 19931231 AND LO_DISCOUNT >= 1 AND LO_DISCOUNT <= 3 AND LO_QUANTITY < 25 - GROUP BY - LO_ORDERKEY;""") + GROUP BY LO_ORDERKEY;""") sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) - VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" - + VALUES + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'), + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'), + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container'), + (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" qt_select_star "select * from lineorder_flat order by 1,2, P_MFGR;" sql "analyze table lineorder_flat with sync;" diff --git a/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy b/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy index 6ead8380eed59b..0ac5f0245ef8ae 100644 --- a/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_2_1/q_2_1.groovy @@ -93,6 +93,8 @@ suite ("mv_ssb_q_2_1") { qt_select_star "select * from lineorder_flat order by 1,2,P_MFGR;" + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');""" + mv_rewrite_success("""SELECT SUM(LO_REVENUE), (LO_ORDERDATE DIV 10000) AS YEAR, P_BRAND diff --git a/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy b/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy index 32765932317701..6b333d97050630 100644 --- a/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_3_1/q_3_1.groovy @@ -66,14 +66,6 @@ suite ("mv_ssb_q_3_1") { ) ENGINE=OLAP DUPLICATE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`) COMMENT "OLAP" - PARTITION BY RANGE(`LO_ORDERDATE`) - (PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 48 PROPERTIES ( "replication_num" = "1", @@ -83,7 +75,8 @@ suite ("mv_ssb_q_3_1") { ); """ - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19920101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' ,'ASIA', 'ASIA', 'MFGR#12', 'MFGR#12', 'brand', 'color', 'type', 4 ,'container');""" + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE, C_NAME, C_ADDRESS, C_CITY, C_NATION, C_REGION, C_PHONE, C_MKTSEGMENT, S_NAME, S_ADDRESS, S_CITY, S_NATION, S_REGION, S_PHONE, P_NAME, P_MFGR, P_CATEGORY, P_BRAND, P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19920101 , 1 , 1 , 1 , 1 , 1 , '1' , 1 , 1 , 1 , 1 , 100 , 1 , 1 , 1 , '2023-06-09' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' , 'ASIA' ,'ASIA', 'ASIA', 'MFGR#12', 'MFGR#12', 'brand', 'color', 'type', 4 ,'container');""" createMV ("""create materialized view lineorder_q_3_1 as SELECT @@ -98,12 +91,16 @@ suite ("mv_ssb_q_3_1") { AND LO_ORDERDATE <= 19971231 GROUP BY C_NATION, S_NATION, YEAR;""") - sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" + sql """INSERT INTO lineorder_flat (LO_ORDERDATE, LO_ORDERKEY, LO_LINENUMBER, LO_CUSTKEY, LO_PARTKEY, LO_SUPPKEY, LO_ORDERPRIORITY, LO_SHIPPRIORITY, LO_QUANTITY, LO_EXTENDEDPRICE, LO_ORDTOTALPRICE, LO_DISCOUNT, LO_REVENUE, LO_SUPPLYCOST, LO_TAX, LO_COMMITDATE, LO_SHIPMODE,C_NAME,C_ADDRESS,C_CITY,C_NATION,C_REGION,C_PHONE,C_MKTSEGMENT,S_NAME,S_ADDRESS,S_CITY,S_NATION,S_REGION,S_PHONE,P_NAME,P_MFGR,P_CATEGORY,P_BRAND,P_COLOR,P_TYPE,P_SIZE,P_CONTAINER) + VALUES (19930101 , 2 , 2 , 2 , 2 , 2 ,'2',2 ,2 ,2 ,2 ,2 ,2 ,2 ,2 ,'2023-06-09','shipmode','name','address','city','nation','region','phone','mktsegment','name','address','city','nation','region','phone','name','mfgr','category','brand','color','type',4,'container');""" qt_select_star "select * from lineorder_flat order by 1,2,P_MFGR;" - explain { - sql("""SELECT + sql """analyze table lineorder_flat with sync;""" + + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');""" + + mv_rewrite_success("""SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, SUM(LO_REVENUE) AS revenue @@ -114,9 +111,9 @@ suite ("mv_ssb_q_3_1") { AND LO_ORDERDATE >= 19920101 AND LO_ORDERDATE <= 19971231 GROUP BY C_NATION, S_NATION, YEAR - ORDER BY YEAR ASC, revenue DESC;""") - contains "(lineorder_q_3_1)" - } + ORDER BY YEAR ASC, revenue DESC;""", + "lineorder_q_3_1") + qt_select_mv """SELECT C_NATION, S_NATION, (LO_ORDERDATE DIV 10000) AS YEAR, diff --git a/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy b/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy index c094e8e00c9eaf..361f60997e3da6 100644 --- a/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_4_1/q_4_1.groovy @@ -92,6 +92,8 @@ suite ("mv_ssb_q_4_1") { sql """analyze table lineorder_flat with sync;""" + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='2');""" + mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit diff --git a/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy b/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy index 1baf21b547f062..ebeb23d96e75e5 100644 --- a/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy +++ b/regression-test/suites/mv_p0/ssb/q_4_1_r1/q_4_1_r1.groovy @@ -119,6 +119,7 @@ suite ("q_4_1_r1") { GROUP BY YEAR, C_NATION ORDER BY YEAR ASC, C_NATION ASC;""" sql """set enable_stats=true;""" + sql """alter table lineorder_flat modify column LO_ORDERDATE set stats ('row_count'='8');""" mv_rewrite_success("""SELECT (LO_ORDERDATE DIV 10000) AS YEAR, C_NATION, SUM(LO_REVENUE - LO_SUPPLYCOST) AS profit diff --git a/regression-test/suites/mv_p0/sum_count/sum_count.groovy b/regression-test/suites/mv_p0/sum_count/sum_count.groovy index b47e51999b7c42..137aabc4e16a89 100644 --- a/regression-test/suites/mv_p0/sum_count/sum_count.groovy +++ b/regression-test/suites/mv_p0/sum_count/sum_count.groovy @@ -51,6 +51,7 @@ suite ("sum_count") { sql "analyze table d_table with sync;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='8');""" mv_rewrite_success("select k1,k4,sum(k2),count(k2) from d_table group by k1,k4;", "kavg") qt_select_mv "select k1,k4,sum(k2),count(k2) from d_table group by k1,k4 order by 1,2;" diff --git a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy index 4e6374e300b09e..85a3f7137b9e40 100644 --- a/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy +++ b/regression-test/suites/mv_p0/sum_divede_count/sum_devide_count.groovy @@ -46,6 +46,7 @@ suite ("sum_devide_count") { sql """analyze table d_table with sync;""" sql """set enable_stats=false;""" + mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;", "kavg") qt_select_mv "select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;" @@ -59,6 +60,7 @@ suite ("sum_devide_count") { qt_select_mv "select sum(k2)/count(k2) from d_table;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from d_table group by k1,k4 order by k1,k4;", "kavg") mv_rewrite_success("select k1,sum(k2)/count(k2) from d_table group by k1 order by k1;", "kavg") diff --git a/regression-test/suites/mv_p0/test_28741/test_28741.groovy b/regression-test/suites/mv_p0/test_28741/test_28741.groovy index 9fabca41b943fc..88fb3c057da092 100644 --- a/regression-test/suites/mv_p0/test_28741/test_28741.groovy +++ b/regression-test/suites/mv_p0/test_28741/test_28741.groovy @@ -69,8 +69,8 @@ suite ("test_28741") { sql """set enable_stats=false;""" mv_rewrite_fail("select b1 from test where t >= '2023-12-20 17:21:00'", "mv_test") - qt_select "select b1 from test where t >= '2023-12-20 17:21:00'" sql """set enable_stats=true;""" + sql """alter table test modify column a set stats ('row_count'='2');""" mv_rewrite_fail("select b1 from test where t >= '2023-12-20 17:21:00'", "mv_test") } diff --git a/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy b/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy index 752cf40d9e5f6a..68952da602aa81 100644 --- a/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy +++ b/regression-test/suites/mv_p0/test_approx_count_distinct/test_approx_count_distinct.groovy @@ -50,6 +50,7 @@ suite ("test_approx_count_distinct") { qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/test_base/test_base.groovy b/regression-test/suites/mv_p0/test_base/test_base.groovy index d2c018febd5ada..97ece8cd6cca68 100644 --- a/regression-test/suites/mv_p0/test_base/test_base.groovy +++ b/regression-test/suites/mv_p0/test_base/test_base.groovy @@ -53,6 +53,7 @@ suite ("test_base") { qt_select_mv "SELECT id,created_at FROM dwd order by 1, 2;" sql """set enable_stats=true;""" + sql """alter table dwd modify column id set stats ('row_count'='2');""" mv_rewrite_success("SELECT created_at, id FROM dwd where created_at = '2020-09-09 00:00:00' order by 1, 2;", "dwd_mv") mv_rewrite_success("SELECT id,created_at FROM dwd where id is not null order by 1, 2;", "dwd_mv") diff --git a/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy b/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy index eac65eeef22adf..0c9e0026cb7fbd 100644 --- a/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy +++ b/regression-test/suites/mv_p0/test_casewhen/test_casewhen.groovy @@ -40,5 +40,6 @@ suite ("test_casewhen") { qt_select_mv "select store_id, sum(case when sale_amt>10 then 1 else 2 end) from sales_records group by store_id order by 1;" sql """set enable_stats=true;""" + sql """alter table sales_records modify column record_id set stats ('row_count'='4');""" mv_rewrite_success("select store_id, sum(case when sale_amt>10 then 1 else 2 end) from sales_records group by store_id order by 1;", "store_amt") } diff --git a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy index 776bf2ecf0bd17..8ee24cb5ba2bd9 100644 --- a/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy +++ b/regression-test/suites/mv_p0/test_create_mv/test_create_mv.groovy @@ -49,6 +49,8 @@ suite("test_create_mv") { sql """ insert into ${tableName} values ('2024-03-20 10:00:00', 'a', 'b', 1) """ + sql """alter table test_mv_10010 modify column load_time set stats ('row_count'='1');""" + sql """ create materialized view mv_1 as select diff --git a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy index 12790500d2f98e..1e07d69ba82871 100644 --- a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy +++ b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy @@ -37,6 +37,8 @@ suite ("create_mv_complex_type") { sql """insert into base_table select 1, 100000, 1.0, '{"jsonk1": 123}', [100, 200], {"k1": 10}, {1, 2};""" + sql """alter table base_table modify column c_int set stats ('row_count'='1');""" + def success = false // 1. special column - mv dup key diff --git a/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy b/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy index 5e99464018e8d7..517b4415bd69ea 100644 --- a/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy +++ b/regression-test/suites/mv_p0/test_doc_e4/test_doc_e4.groovy @@ -57,6 +57,7 @@ suite ("test_doc_e4") { qt_select_mv "select year(k4)+month(k4) from d_table where year(k4) = 2020 order by 1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='3');""" mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from d_table group by abs(k1)+k2+1 order by 1,2;", "k1a2p2ap3ps") mv_rewrite_success("select bin(abs(k1)+k2+1),sum(abs(k2+2)+k3+3) from d_table group by bin(abs(k1)+k2+1);", "k1a2p2ap3ps") diff --git a/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy b/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy index 5e917e9b0f5bd0..23aeeb207f6898 100644 --- a/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy +++ b/regression-test/suites/mv_p0/test_dup_group_by_mv_abs/test_dup_group_by_mv_abs.groovy @@ -53,6 +53,7 @@ suite ("test_dup_group_by_mv_abs") { qt_select_mv_sub "select sum(abs(k2)) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(abs(k2)) from d_table group by k1;", "k12sa") mv_rewrite_success("select sum(abs(k2)) from d_table group by k1;", "k12sa") } diff --git a/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy index ba9f60f6df5026..3551bb8a7fb359 100644 --- a/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy +++ b/regression-test/suites/mv_p0/test_dup_group_by_mv_plus/test_dup_group_by_mv_plus.groovy @@ -53,6 +53,7 @@ suite ("test_dup_group_by_mv_plus") { qt_select_mv_sub "select sum(k2+1) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(k2+1) from d_table group by k1;", "k12sp") mv_rewrite_success("select sum(k2+1) from d_table group by k1;", "k12sp") diff --git a/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy b/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy index 89088ffb9ea317..f44c9d134f91f0 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_abs/test_dup_mv_abs.groovy @@ -65,6 +65,7 @@ suite ("test_dup_mv_abs") { qt_select_group_mv_not "select sum(abs(k2)) from d_table group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,abs(k2) from d_table order by k1;", "k12a") mv_rewrite_success("select abs(k2) from d_table order by k1;", "k12a") diff --git a/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy b/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy index a7db2617fe6067..4b8fa5aa6359e8 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_bin/test_dup_mv_bin.groovy @@ -64,6 +64,7 @@ suite ("test_dup_mv_bin") { qt_select_group_mv_not "select group_concat(bin(k2)) from d_table group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,bin(k2) from d_table order by k1;", "k12b") mv_rewrite_success("select bin(k2) from d_table order by k1;", "k12b") diff --git a/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy b/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy index 659f44de1e93c4..6530fd84cac7a8 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_bitmap_hash/test_dup_mv_bitmap_hash.groovy @@ -46,6 +46,7 @@ suite ("test_dup_mv_bitmap_hash") { mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;", "k1g2bm") qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from d_table group by k1 order by k1;", "k1g2bm") createMV "create materialized view k1g3bm as select k1,bitmap_union(bitmap_hash(k3)) from d_table group by k1;" @@ -58,11 +59,13 @@ suite ("test_dup_mv_bitmap_hash") { qt_select_star "select * from d_table order by k1,k2,k3;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" sql """analyze table d_table with sync;""" sql """set enable_stats=false;""" mv_rewrite_success("select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1;", "k1g3bm") qt_select_mv_sub "select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,bitmap_union_count(bitmap_hash(k3)) from d_table group by k1;", "k1g3bm") } diff --git a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy index 5abe03e736b641..ad437db464b3bb 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy @@ -70,7 +70,7 @@ suite ("test_dup_mv_plus") { qt_select_mv "select k1,k2+1 from d_table order by k2;" sql """set enable_stats=true;""" - + sql """alter table d_table modify column k4 set stats ('row_count'='3');""" mv_rewrite_success("select k1,k2+1 from d_table order by k1;", "k12p") mv_rewrite_success("select k2+1 from d_table order by k1;", "k12p") diff --git a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy index 562fba25b2e8a9..5ccdae1bd8a7c5 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_repeat/test_dup_mv_repeat.groovy @@ -47,6 +47,7 @@ suite ("test_dup_mv_repeat") { qt_select_mv "SELECT s AS s, sum(n) / count(DISTINCT dt) AS n FROM db1 GROUP BY GROUPING SETS((s)) order by 1;" sql """set enable_stats=true;""" + sql """alter table db1 modify column dt set stats ('row_count'='2');""" mv_rewrite_success("SELECT s AS s, sum(n) / count(DISTINCT dt) AS n FROM db1 GROUP BY GROUPING SETS((s)) order by 1;", "dbviwe") } diff --git a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy index 92a8d6520f3122..18f798f8b6aac3 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy @@ -45,6 +45,7 @@ suite ("test_dup_mv_year") { qt_select_mv "select k1,year(k2) from d_table order by k1;" sql """set enable_stats=true;""" + sql """alter table d_table modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,year(k2) from d_table order by k1;", "k12y") createMV "create materialized view k13y as select k1,year(k3) from d_table;" diff --git a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy index 50f1fd61806f2e..2fadc0b0da91a0 100644 --- a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy +++ b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy @@ -56,6 +56,7 @@ suite ("test_mv_dp") { time 10000 // limit inflight 10s } */ + mv_rewrite_success("""select d, bitmap_union_count(bitmap_from_array(cast(uid_list as array))), bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) @@ -68,6 +69,7 @@ suite ("test_mv_dp") { from dp group by d order by 1;""" sql """set enable_stats=true;""" + sql """alter table dp modify column d set stats ('row_count'='4');""" mv_rewrite_success("""select d, bitmap_union_count(bitmap_from_array(cast(uid_list as array))), bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) diff --git a/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy b/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy index b9c79f8bc11635..35d082fbff6082 100644 --- a/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy +++ b/regression-test/suites/mv_p0/test_mv_mor/test_mv_mor.groovy @@ -39,6 +39,8 @@ suite ("test_mv_mor") { sql "insert into u_table select 1,1,1,2;" sql "insert into u_table select 1,2,1,2;" + sql """alter table u_table modify column k1 set stats ('row_count'='2');""" + // do not match mv coz preagg is off, mv need contains all key column to make row count correct mv_rewrite_fail("select k1,k2+k3 from u_table order by k1;", "k123p") qt_select_mv "select k1,k2+k3 from u_table order by k1;" diff --git a/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy b/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy index d2c9921edefe7a..2d43785ba6fafb 100644 --- a/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy +++ b/regression-test/suites/mv_p0/test_ndv/test_ndv.groovy @@ -50,6 +50,7 @@ suite ("test_ndv") { qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy b/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy index c6b1dd2d8a9b1c..12f5765666ef28 100644 --- a/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy +++ b/regression-test/suites/mv_p0/test_nvl/test_nvl.groovy @@ -51,6 +51,7 @@ suite ("test_nvl") { qt_select_mv "select ifnull(id,0) from dwd order by 1;" sql """set enable_stats=true;""" + sql """alter table dwd modify column id set stats ('row_count'='2');""" mv_rewrite_success("select nvl(id,0) from dwd order by 1;", "dwd_mv") mv_rewrite_success("select ifnull(id,0) from dwd order by 1;", "dwd_mv") diff --git a/regression-test/suites/mv_p0/test_o2/test_o2.groovy b/regression-test/suites/mv_p0/test_o2/test_o2.groovy index 665156056c0096..34d31a08cfae7b 100644 --- a/regression-test/suites/mv_p0/test_o2/test_o2.groovy +++ b/regression-test/suites/mv_p0/test_o2/test_o2.groovy @@ -60,6 +60,7 @@ suite ("test_o2") { qt_select_mv "select ts,metric_name,platform,sum(count_value) from o2_order_events group by ts,metric_name,platform;" sql """set enable_stats=true;""" + sql """alter table o2_order_events modify column ts set stats ('row_count'='2');""" mv_rewrite_success("select ts,metric_name,platform,sum(count_value) from o2_order_events group by ts,metric_name,platform;", "o2_order_events_mv") } diff --git a/regression-test/suites/mv_p0/test_substr/test_substr.groovy b/regression-test/suites/mv_p0/test_substr/test_substr.groovy index c43c3ae6483acd..7d6ae82634fd8c 100644 --- a/regression-test/suites/mv_p0/test_substr/test_substr.groovy +++ b/regression-test/suites/mv_p0/test_substr/test_substr.groovy @@ -56,6 +56,7 @@ suite ("test_substr") { qt_select_mv "SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd group by substr(created_at,1,10);" sql """set enable_stats=true;""" + sql """alter table dwd modify column id set stats ('row_count'='2');""" mv_rewrite_success("SELECT substr(created_at,1,10) as statistic_date, max(dt) as dt FROM dwd group by substr(created_at,1,10);", "dwd_mv") } diff --git a/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy b/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy index f65e36484e7de9..bbfa8ea6a54d17 100644 --- a/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy +++ b/regression-test/suites/mv_p0/test_tbl_name/test_tbl_name.groovy @@ -66,6 +66,7 @@ suite ("test_tbl_name") { group by id order by 1,2; """ sql """set enable_stats=true;""" + sql """alter table functionality_olap modify column id set stats ('row_count'='2');""" mv_rewrite_success("""select functionality_olap.id as id, sum(functionality_olap.score) as score_max diff --git a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy index b62ec696224197..69264d838e4d24 100644 --- a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy +++ b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy @@ -62,6 +62,7 @@ suite ("test_upper_alias") { qt_select_mv "SELECT d_a AS d_b FROM test_0401 order by 1;" sql """set enable_stats=true;""" + sql """alter table test_0401 modify column d_b set stats ('row_count'='3');""" mv_rewrite_any_success("SELECT upper(d_b) AS d_b FROM test_0401 GROUP BY upper(d_b) order by 1;", ["test_0401_mv", "test_0401_mv2"]) diff --git a/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy b/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy index 3d139cbc01bbe7..f971673971a2aa 100644 --- a/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy +++ b/regression-test/suites/mv_p0/test_user_activity/test_user_activity.groovy @@ -53,6 +53,7 @@ suite ("test_user_activity") { qt_select_group_mv "select n_dx, percentile_approx(n_duration, 0.5) as p50, percentile_approx(n_duration, 0.90) as p90 FROM u_axx GROUP BY n_dx;" sql """set enable_stats=true;""" + sql """alter table u_axx modify column r_xx set stats ('row_count'='3');""" mv_rewrite_success("select n_dx, percentile_approx(n_duration, 0.5) as p50, percentile_approx(n_duration, 0.90) as p90 FROM u_axx GROUP BY n_dx;", "session_distribution_2") } diff --git a/regression-test/suites/mv_p0/unique/unique.groovy b/regression-test/suites/mv_p0/unique/unique.groovy index 1209d587c78941..d54329a5c54e61 100644 --- a/regression-test/suites/mv_p0/unique/unique.groovy +++ b/regression-test/suites/mv_p0/unique/unique.groovy @@ -73,6 +73,7 @@ suite ("unique") { qt_select_star "select * from u_table order by k1;" sql """set enable_stats=true;""" + sql """alter table u_table modify column k1 set stats ('row_count'='3');""" mv_rewrite_success("select k3,length(k1),k2 from u_table order by 1,2,3;", "k31l42") // todo: support match query diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy index 70b016de2da80f..f39e8df6cabe6a 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy @@ -36,6 +36,7 @@ suite ("testAggQueryOnAggMV1") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" +sql """alter table emps modify column time_col set stats ('row_count'='4');""" createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno;") createMV("create materialized view emps_mv_count_key as select deptno, count(deptno) from emps group by deptno;") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy index a25ac24404e181..739dde33372cb6 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy @@ -51,6 +51,7 @@ suite ("testAggQueryOnAggMV10") { qt_select_mv "select deptno, commission, sum(salary) + 1 from emps group by rollup (deptno, commission) order by 1,2;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select deptno, commission, sum(salary) + 1 from emps group by rollup (deptno, commission);", diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy index c26db8e034e66a..a4d69136bc03ca 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy @@ -50,6 +50,7 @@ suite ("testAggQueryOnAggMV11") { qt_select_mv "select deptno, count(salary) + count(1) from emps group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_fail("select deptno, count(salary) + count(1) from emps group by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy index 945e4e9194910f..5cf7d257fe8b73 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy @@ -53,6 +53,7 @@ suite ("testAggQueryOnAggMV2") { qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where (sum_salary * 2) > 3 order by deptno ;", diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy index 46457f0d0a81d4..d2f61e04762b2f 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy @@ -38,7 +38,6 @@ suite ("testAggQueryOnAggMV3") { sql """insert into emps values("2020-01-04",4,"d",21,4,4);""" - createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;") sql "analyze table emps with sync;" @@ -56,6 +55,7 @@ suite ("testAggQueryOnAggMV3") { qt_select_mv "select commission, sum(salary) from emps where commission = 100 group by commission order by commission;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select commission, sum(salary) from emps where deptno > 0 and commission * (deptno + commission) = 100 group by commission order by commission;", diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy index 1fa4342952cad4..1f14728e3b7b2c 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy @@ -51,6 +51,7 @@ suite ("testAggQuqeryOnAggMV5") { qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where sum_salary>10 order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps group by deptno) a where sum_salary>0;", diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy index 3cca0273051f0a..4cafeaa185f4a3 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV6/testAggQuqeryOnAggMV6.groovy @@ -51,6 +51,7 @@ suite ("testAggQuqeryOnAggMV6") { qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from emps where deptno>=20 group by deptno) a where sum_salary>10 order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from emps where deptno>=0 group by deptno) a where sum_salary>10;", diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy index 1a606141e38f04..d43b5796e6d7f9 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy @@ -50,6 +50,8 @@ suite ("testAggQuqeryOnAggMV7") { qt_select_mv "select deptno, sum(salary) from emps where deptno>=20 group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select deptno, sum(salary) from emps where deptno>=20 group by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy index 7f2427e275de47..6dc78784956d32 100644 --- a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy @@ -50,6 +50,7 @@ suite ("testAggregateMVCalcAggFunctionQuery") { qt_select_mv "select deptno, sum(salary + 1) from emps where deptno > 10 group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_fail("select deptno, sum(salary + 1) from emps where deptno > 10 group by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy index 69410ae7be5d0d..c1287bfa94bcc4 100644 --- a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy @@ -51,7 +51,7 @@ suite ("testBitmapUnionInQuery") { qt_select_mv "select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;" sql """set enable_stats=true;""" - + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from user_tags group by user_id having a>1 order by a;", diff --git a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy index 12152aad369e1d..5cc05db62be1a7 100644 --- a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy +++ b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy @@ -66,6 +66,10 @@ suite ("testCountDistinctToBitmap") { sql """insert into user_tags2 values("2020-01-01",1,"a",1);""" sql """insert into user_tags2 values("2020-01-02",2,"b",2);""" + + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags2 modify column time_col set stats ('row_count'='3');""" + createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags2 group by user_id;") sql """insert into user_tags2 values("2020-01-01",1,"a",2);""" diff --git a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy index c671e106a10648..698490f0943e88 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy @@ -47,6 +47,7 @@ suite ("testIncorrectMVRewriteInSubquery") { qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_fail("select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags where user_name in (select user_name from user_tags group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;", diff --git a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy index dbece0c9592364..45230c5b5587d8 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy @@ -45,6 +45,7 @@ suite ("testIncorrectRewriteCountDistinct") { qt_select_mv "select user_name, count(distinct tag_id) from user_tags group by user_name order by user_name;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_fail("select user_name, count(distinct tag_id) from user_tags group by user_name;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy index 3caf0000c1824d..8f41fc1ffb5e74 100644 --- a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy +++ b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy @@ -43,6 +43,8 @@ suite ("testJoinOnLeftProjectToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table depts modify column time_col set stats ('row_count'='3');""" + sql """insert into depts values("2020-01-02",2,"b",2);""" sql """insert into depts values("2020-01-03",3,"c",3);""" sql """insert into depts values("2020-01-02",2,"b",1);""" @@ -59,6 +61,7 @@ suite ("testJoinOnLeftProjectToJoin") { qt_select_mv "select * from (select deptno , sum(salary) from emps group by deptno) A join (select deptno, max(cost) from depts group by deptno ) B on A.deptno = B.deptno order by A.deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_all_success("select * from (select deptno , sum(salary) from emps group by deptno) A join (select deptno, max(cost) from depts group by deptno ) B on A.deptno = B.deptno;", ["emps_mv", "depts_mv"]) } diff --git a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy index a9456cb4e79c96..c77f58609ca0cb 100644 --- a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy +++ b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy @@ -50,6 +50,7 @@ suite ("testNDVToHll") { qt_select_mv "select user_id, approx_count_distinct(tag_id) a from user_tags group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") mv_rewrite_success("select user_id, ndv(tag_id) a from user_tags group by user_id order by user_id;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy index 2aa88d84f3d433..41cf480e880cd0 100644 --- a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy +++ b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy @@ -49,6 +49,7 @@ suite ("testOrderByQueryOnProjectView") { qt_select_mv "select empid from emps order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid from emps where deptno > 0 order by deptno;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy index d95c3e2fc6de88..d578cbddba945d 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy @@ -61,6 +61,7 @@ suite ("testProjectionMV1") { qt_select_mv "select deptno, sum(empid) from emps group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid, deptno from emps where deptno > 0 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy index 2f7548b14c8635..b663bfcd2b1738 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy @@ -51,6 +51,7 @@ suite ("testProjectionMV2") { qt_select_base "select name from emps where deptno -1 = 0 order by empid;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid + 1 from emps where deptno = 1 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy index e4ea1aa4de023e..378ec80d6cfd72 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy @@ -53,6 +53,7 @@ suite ("testProjectionMV3") { qt_select_mv2 "select name from emps where deptno = 1 order by empid;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_success("select empid + 1, name from emps where deptno = 1 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy index 0c6f737deabb29..37a57f80d03cae 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy @@ -52,6 +52,7 @@ suite ("testProjectionMV4") { qt_select_base "select empid from emps where deptno > 1 and empid > 1 order by empid;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") mv_rewrite_fail("select empid from emps where deptno > 1 and empid > 1 order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy index 57ddba4d3c8f72..15394870f9b11c 100644 --- a/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy +++ b/regression-test/suites/mv_p0/ut/testQueryOnStar/testQueryOnStar.groovy @@ -70,4 +70,5 @@ suite ("testQueryOnStar") { contains "(emps_mv)" } qt_select_mv "select ref_1.`empid` as c0 from tpch_tiny_region as ref_0 left join emps as ref_1 on (ref_0.`r_comment` = ref_1.`name` ) where true order by ref_0.`r_regionkey`,ref_0.`r_regionkey` desc ,ref_0.`r_regionkey`,ref_0.`r_regionkey`;" + } diff --git a/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy b/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy index 0cfd80fcb06b4e..9e3bd886b31ea5 100644 --- a/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy +++ b/regression-test/suites/mv_p0/ut/testSelectMVWithTableAlias/testSelectMVWithTableAlias.groovy @@ -29,6 +29,8 @@ suite ("testSelectMVWithTableAlias") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" diff --git a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy index 006b72885ec112..73ddb21109bb91 100644 --- a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy +++ b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy @@ -52,6 +52,7 @@ suite ("testSingleMVMultiUsage") { } qt_select_mv "select * from (select deptno, empid from emps where deptno>100) A join (select deptno, empid from emps where deptno >200) B using (deptno) order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") explain { diff --git a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy index 6c1747059d1ae3..cd134a5d48ea40 100644 --- a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy @@ -38,7 +38,6 @@ suite ("testSubQuery") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view emps_mv as select deptno, empid from emps;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" @@ -52,5 +51,6 @@ suite ("testSubQuery") { qt_select_mv "select empid, deptno, salary from emps e1 where empid = (select max(empid) from emps where deptno = e1.deptno) order by deptno;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") } diff --git a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy index f504c81f214e91..c6243a3f923616 100644 --- a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy +++ b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy @@ -35,6 +35,7 @@ suite ("testUnionDistinct") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + createMV("create materialized view emps_mv as select empid, deptno from emps order by empid, deptno;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" @@ -52,6 +53,7 @@ suite ("testUnionDistinct") { } qt_select_mv "select * from (select empid, deptno from emps where empid >1 union select empid, deptno from emps where empid <0) t order by 1;" sql """set enable_stats=true;""" + sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") explain { diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy index c51b7cc95d201b..f643f551af83d5 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_on_none_agg/agg_on_none_agg.groovy @@ -57,6 +57,8 @@ suite("agg_on_none_agg") { (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); """ + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """ drop table if exists lineitem """ @@ -95,6 +97,8 @@ suite("agg_on_none_agg") { (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """ drop table if exists partsupp """ @@ -120,6 +124,8 @@ suite("agg_on_none_agg") { (2, 3, 10, 11.01, 'supply2'); """ + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + sql """analyze table orders with sync;""" sql """analyze table lineitem with sync;""" sql """analyze table partsupp with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy index 050662e19190c4..27a8b693dfe9ac 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_variety/agg_variety.groovy @@ -95,6 +95,8 @@ suite("agg_variety") { (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); """ + + sql """ drop table if exists partsupp """ @@ -124,6 +126,10 @@ suite("agg_variety") { sql """analyze table lineitem with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // query dimension is less then mv def mv1_0 = """ select diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index cda9b21fc4e51d..5b8aa6e00d2308 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -132,6 +132,9 @@ suite("aggregate_with_roll_up") { sql """analyze table partsupp with sync""" sql """analyze table lineitem with sync""" sql """analyze table orders with sync""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" // multi table // filter inside + left + use roll up dimension diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 85f85101961a97..83f924a7ea5c65 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -124,6 +124,12 @@ suite("aggregate_without_roll_up") { (2, 3, 10, 11.01, 'supply2', null); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // single table // with filter def mv1_0 = "select o_shippriority, o_comment, " + diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy index 5a6c75f6c59c6e..b9e671f2636b82 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy @@ -49,6 +49,7 @@ suite("grace_period") { "replication_num" = "1" ); """ + sql """alter table orders_partition modify column o_comment set stats ('row_count'='3');""" sql """ drop table if exists lineitem_partition @@ -81,6 +82,7 @@ suite("grace_period") { "replication_num" = "1" ); """ + sql """alter table lineitem_partition modify column l_comment set stats ('row_count'='3');""" sql """ insert into orders_partition values diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy index 72a8a691078a2d..4af1d778c002b2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy @@ -120,6 +120,15 @@ suite("materialized_view_switch") { (2, 3, 10, 11.01, 'supply2'); """ + sql """analyze table lineitem with sync; + analyze table orders with sync; + analyze table partsupp with sync; + """ + + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + def mv_name = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey from lineitem @@ -134,12 +143,17 @@ suite("materialized_view_switch") { where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5; """ - async_mv_rewrite_success(db, mv_name, query, "mv_name") + async_mv_rewrite_success(db, mv_name, query, "mv_name_1") + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name_1""" + sql "SET enable_materialized_view_rewrite=false" - mv_not_part_in(query, "mv_name") + create_async_mv(db, "mv_name_2", mv_name) + mv_not_part_in(query, "mv_name_2") + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name_2""" + sql "SET enable_materialized_view_rewrite=true" - async_mv_rewrite_success(db, mv_name, query, "mv_name") - sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name""" + async_mv_rewrite_success(db, mv_name, query, "mv_name_3") + sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name_3""" // test when materialized_view_relation_mapping_max_count is 8 def mv1_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy index f2b5ad09c0b17a..3aed3b0f9e24df 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy @@ -98,6 +98,9 @@ suite("partition_mv_rewrite_dimension_1") { (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); """ + sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');""" + sql """analyze table orders_1 with sync;""" sql """analyze table lineitem_1 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy index 53f8777a1589d7..c7ee359cdef2e4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_3") { sql """analyze table orders_2_3 with sync;""" sql """analyze table lineitem_2_3 with sync;""" + sql """alter table orders_2_3 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_3 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy index deb6255119a26e..e59b2771dd4e57 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy @@ -124,6 +124,10 @@ suite("partition_mv_rewrite_dimension_2_4") { (3, null, 1, 99.5, 'yy'); """ + sql """alter table orders_2_4 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_4 modify column l_comment set stats ('row_count'='7');""" + sql """alter table partsupp_2_4 modify column ps_comment set stats ('row_count'='3');""" + sql """analyze table orders_2_4 with sync;""" sql """analyze table lineitem_2_4 with sync;""" sql """analyze table partsupp_2_4 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy index 5d8a6b5270285b..e5e43073bede3e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy @@ -124,6 +124,10 @@ suite("partition_mv_rewrite_dimension_2_5") { (3, null, 1, 99.5, 'yy'); """ + sql """alter table orders_2_5 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_5 modify column l_comment set stats ('row_count'='7');""" + sql """alter table partsupp_2_5 modify column ps_comment set stats ('row_count'='3');""" + sql """analyze table orders_2_5 with sync;""" sql """analyze table lineitem_2_5 with sync;""" sql """analyze table partsupp_2_5 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy index 1831e8de9a7a9d..f3289b007a0505 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy @@ -124,6 +124,12 @@ suite("partition_mv_rewrite_dimension_2_6") { (3, null, 1, 99.5, 'yy'); """ + sql """alter table orders_2_6 modify column o_comment set stats ('row_count'='10');""" + + sql """alter table lineitem_2_6 modify column l_comment set stats ('row_count'='7');""" + + sql """alter table partsupp_2_6 modify column ps_comment set stats ('row_count'='3');""" + sql """analyze table orders_2_6 with sync;""" sql """analyze table lineitem_2_6 with sync;""" sql """analyze table partsupp_2_6 with sync;""" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy index 6def436e11c5c4..e36a5e277daca7 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy @@ -102,6 +102,8 @@ suite("partition_mv_rewrite_dimension_2_full_join") { sql """analyze table orders_2_full_join with sync;""" sql """analyze table lineitem_2_full_join with sync;""" + sql """alter table orders_2_full_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_full_join modify column l_comment set stats ('row_count'='7');""" def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy index 526e05f0692fbe..a615c7316bdb56 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_2") { sql """analyze table orders_2_2 with sync;""" sql """analyze table lineitem_2_2 with sync;""" + sql """alter table orders_2_2 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_2 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy index 0db7b3bf008c68..35d48270574a6f 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_left_anti_join") { sql """analyze table orders_2_left_anti_join with sync;""" sql """analyze table lineitem_2_left_anti_join with sync;""" + sql """alter table orders_2_left_anti_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_left_anti_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy index f59dffd0091bb4..616707412954e4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_1") { sql """analyze table orders_2_1 with sync;""" sql """analyze table lineitem_2_1 with sync;""" + sql """alter table orders_2_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_1 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy index 5e60250ef91d26..f203db1134f2f2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_left_semi_join") { sql """analyze table orders_2_left_semi_join with sync;""" sql """analyze table lineitem_2_left_semi_join with sync;""" + sql """alter table orders_2_left_semi_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_left_semi_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy index 289ed57287f1d8..c6097d845f41c9 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_anti_join") { sql """analyze table orders_2_right_anti_join with sync;""" sql """analyze table lineitem_2_right_anti_join with sync;""" + sql """alter table orders_2_right_anti_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_right_anti_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy index 8d6abb6dc561f8..850de42fc696eb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_join") { sql """analyze table orders_2_right_join with sync;""" sql """analyze table lineitem_2_right_join with sync;""" + sql """alter table orders_2_right_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_right_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy index 8f1f6f48a0ae91..97ddcd0eb1da29 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy @@ -102,6 +102,9 @@ suite("partition_mv_rewrite_dimension_2_right_semi_join") { sql """analyze table orders_2_right_semi_join with sync;""" sql """analyze table lineitem_2_right_semi_join with sync;""" + sql """alter table orders_2_right_semi_join modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_2_right_semi_join modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy index f433e18f48c062..776f1b996f1826 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy @@ -101,6 +101,9 @@ suite("partition_mv_rewrite_dimension_self_conn") { sql """analyze table orders_self_conn with sync;""" sql """analyze table lineitem_self_conn with sync;""" + sql """alter table orders_self_conn modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_self_conn modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy index dee43f20f4926a..fd41789d02661e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy @@ -101,6 +101,9 @@ suite("dimension_2_join_agg_replenish") { sql """analyze table orders with sync;""" sql """analyze table lineitem with sync;""" + sql """alter table orders modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt @@ -649,19 +652,19 @@ suite("dimension_2_join_agg_replenish") { """ def right_semi_mv_stmt_8 = """select t1.l_orderkey, t1.l_shipdate, t1.l_partkey from orders - right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1 from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 on t1.l_orderkey = orders.o_orderkey group by t1.l_orderkey, t1.l_shipdate, t1.l_partkey """ def right_semi_mv_stmt_9 = """select t1.l_orderkey, t1.l_shipdate, t1.l_partkey from orders - right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem where l_orderkey > 1 + 1 group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 + right semi join (select l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate, sum(l_quantity) as col1, count(*) as col2 from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey, l_partkey, l_suppkey, l_quantity, l_shipdate) as t1 on t1.l_orderkey = orders.o_orderkey group by t1.l_orderkey, t1.l_shipdate, t1.l_partkey """ - def right_semi_mv_stmt_10 = """select t1.l_orderkey + def right_semi_mv_stmt_10 = """select t1.sum_total, max_total+min_total as col3, count_all from orders right semi join (select l_orderkey, sum(l_quantity) as sum_total, max(l_quantity) as max_total, @@ -670,10 +673,10 @@ suite("dimension_2_join_agg_replenish") { sum(l_quantity) + count(*) as col5, bitmap_union(to_bitmap(case when l_quantity > 1 and l_orderkey IN (1, 3) then l_partkey else null end)) as cnt_1, bitmap_union(to_bitmap(case when l_quantity > 2 and l_orderkey IN (2) then l_partkey else null end)) as cnt_2 - from lineitem where l_orderkey > 1 + 1 group by l_orderkey) as t1 + from lineitem where l_shipdate >= "2023-10-17" group by l_orderkey) as t1 on t1.l_orderkey = orders.o_orderkey group by - t1.l_orderkey + t1.sum_total, col3, count_all """ def left_anti_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey @@ -790,7 +793,7 @@ suite("dimension_2_join_agg_replenish") { bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 from orders where o_orderkey >= 1 + 1 group by o_orderkey) as t1 right anti join lineitem on lineitem.l_orderkey = t1.o_orderkey - group by l_orderkey, l_shipdate, l_partkey + group by l_orderkey, l_shipdate, l_partkey """ def right_anti_mv_stmt_6 = """select t1.l_shipdate, t1.l_quantity, t1.l_orderkey diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy index 04bd6ac5c6ac8d..3afbe42fd2a1c3 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_join_agg_negative.groovy @@ -101,6 +101,9 @@ suite("dimension_join_agg_negative") { sql """analyze table orders_negative with sync;""" sql """analyze table lineitem_negative with sync;""" + sql """alter table orders_negative modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_negative modify column l_comment set stats ('row_count'='7');""" + // left join + agg (function + group by + +-*/ + filter) def left_mv_stmt_1 = """select t1.o_orderdate, t1.o_shippriority, t1.o_orderkey from (select o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority from orders_negative group by o_orderkey, o_custkey, o_orderstatus, o_orderdate, o_shippriority) as t1 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy index 763880ea94be44..2ca4a524989c99 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy @@ -99,6 +99,9 @@ suite("filter_equal_or_notequal_case") { sql """analyze table orders_1 with sync;""" sql """analyze table lineitem_1 with sync;""" + sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy index ccab3aaf35360c..80645f6cebf013 100644 --- a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy @@ -131,6 +131,9 @@ suite("direct_query_mv") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='15');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" create_async_mv(db, "mv1_0", """ diff --git a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy index ec8cecbaf517d0..4655f9d004bc7a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy @@ -127,6 +127,9 @@ suite("materialized_view_grouping_sets") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table orders modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='7');""" + // query has group sets, and mv doesn't // single table grouping sets without grouping scalar function def mv1_0 = diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy index 2b018ebc37373b..8c0674faeb8c32 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy @@ -123,6 +123,10 @@ suite("inner_join_dphyp") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // without filter def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from lineitem " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy index 45d9aee067c798..71154fbeb276cf 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy @@ -123,6 +123,10 @@ suite("outer_join_dphyp") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // without filter def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from lineitem " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy index b745f7d0197c28..3aebf92b7cec71 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy @@ -122,6 +122,10 @@ suite("inner_join") { sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // without filter def mv1_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy index daebc78f23457c..7ec650c2ee2ddd 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy @@ -278,9 +278,20 @@ suite("outer_join") { sql """analyze table lineitem with sync;""" sql """analyze table orders with sync;""" sql """analyze table partsupp with sync;""" + sql """analyze table lineitem_null with sync;""" + sql """analyze table orders_null with sync;""" sql """analyze table orders_same_col with sync;""" sql """analyze table lineitem_same_col with sync;""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table lineitem_same_col modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table orders_same_col modify column o_comment set stats ('row_count'='18');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + sql """alter table lineitem_null modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders_null modify column o_comment set stats ('row_count'='5');""" + + // without filter def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + "from lineitem " + diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy index 5688c2bbe21d6a..cce090b2a940fb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/inner_join_infer_and_derive.groovy @@ -106,6 +106,9 @@ suite("inner_join_infer_and_derive") { sql """analyze table orders_inner with sync;""" sql """analyze table lineitem_inner with sync;""" + sql """alter table orders_inner modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_inner modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy index dbfaa783383af8..7e51847c3b1a07 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/left_join_infer_and_derive.groovy @@ -106,6 +106,8 @@ suite("left_join_infer_and_derive") { sql """analyze table orders_left with sync;""" sql """analyze table lineitem_left with sync;""" + sql """alter table orders_left modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_left modify column l_comment set stats ('row_count'='7');""" def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy index 4b5bdf467a5d02..1761efa5ef4475 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join_infer_derive/right_join_infer_and_derive.groovy @@ -106,6 +106,9 @@ suite("right_join_infer_and_derive") { sql """analyze table orders_right with sync;""" sql """analyze table lineitem_right with sync;""" + sql """alter table orders_right modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_right modify column l_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt diff --git a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy index 455ab30d11611f..6eeeff0ae430b6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy @@ -127,6 +127,9 @@ suite("negative_partition_mv_rewrite") { sql """analyze table lineitem_1 with sync;""" sql """analyze table partsupp_1 with sync;""" + sql """alter table orders_1 modify column o_comment set stats ('row_count'='10');""" + sql """alter table lineitem_1 modify column l_comment set stats ('row_count'='7');""" + def mv_name = "mv_1" def mtmv_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy index 942db734543330..5bb8c473c03d2a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -199,6 +199,10 @@ suite("nested_materialized_view") { (2, 3, 10, 11.01, 'supply2'); """ + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // simple nested materialized view def mv1_0_inner_mv = """ select diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy index 4dd61a2150cc8c..4870ec99e659c0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy @@ -96,12 +96,20 @@ suite("nested_mtmv") { insert into orders_1 values (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), + (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), + (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); """ @@ -109,22 +117,29 @@ suite("nested_mtmv") { sql """ insert into lineitem_1 values (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (1, 1, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (3, 3, 3, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (3, 3, 3, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 2, 3, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (2, 1, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), + (2, 1, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), (3, 1, 3, 1, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 2, 1, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), (2, 2, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), (3, 3, 3, 3, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (3, 3, 3, 3, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), (1, 1, 1, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); """ sql""" insert into partsupp_1 values (1, 1, 1, 99.5, 'yy'), + (1, 1, 1, 99.5, 'yy'), + (2, 2, 2, 109.2, 'mm'), (2, 2, 2, 109.2, 'mm'), (3, 3, 1, 99.5, 'yy'), + (3, 3, 1, 99.5, 'yy'), (3, null, 1, 99.5, 'yy'); """ @@ -132,6 +147,10 @@ suite("nested_mtmv") { sql """analyze table lineitem_1 with sync;""" sql """analyze table partsupp_1 with sync;""" + sql """alter table orders_1 modify column o_orderdate set stats ('row_count'='17');""" + sql """alter table lineitem_1 modify column l_shipdate set stats ('row_count'='14');""" + sql """alter table partsupp_1 modify column ps_comment set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" def origin_res = sql stmt @@ -659,155 +678,77 @@ suite("nested_mtmv") { left join ( select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 from ( - select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, + t.agg1 as agg1, + t.sum_total as agg3, + t.max_total as agg4, + t.min_total as agg5, + t.count_all as agg6, + cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t1 - left join ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t2 - on t1.l_orderkey = t2.l_orderkey - where t1.l_orderkey > 1 - group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from lineitem_1 + inner join orders_1 + on lineitem_1.l_orderkey = orders_1.o_orderkey + where lineitem_1.l_shipdate >= "2023-10-17" + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey + ) as t + inner join partsupp_1 + on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey + where partsupp_1.ps_suppkey > 1 + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 ) as t1 left join ( - select t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, + t.agg1 as agg1, + t.sum_total as agg3, + t.max_total as agg4, + t.min_total as agg5, + t.count_all as agg6, + cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t1 - left join ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, - t.agg1 as agg1, - t.sum_total as agg3, - t.max_total as agg4, - t.min_total as agg5, - t.count_all as agg6, - cast(sum(IFNULL(ps_suppkey, 0) * IFNULL(ps_partkey, 0)) as decimal(28, 8)) as agg2 - from ( - select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, - sum(o_totalprice) as sum_total, - max(o_totalprice) as max_total, - min(o_totalprice) as min_total, - count(*) as count_all, - bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, - bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 - from lineitem_1 - inner join orders_1 - on lineitem_1.l_orderkey = orders_1.o_orderkey - where lineitem_1.l_shipdate >= "2023-10-17" - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey - ) as t - inner join partsupp_1 - on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey - where partsupp_1.ps_suppkey > 1 - group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 - ) as t2 - on t1.l_orderkey = t2.l_orderkey - where t1.l_orderkey > 1 - group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 + select l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, cast(sum(IFNULL(o_orderkey, 0) * IFNULL(o_custkey, 0)) as decimal(28, 8)) as agg1, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from lineitem_1 + inner join orders_1 + on lineitem_1.l_orderkey = orders_1.o_orderkey + where lineitem_1.l_shipdate >= "2023-10-17" + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey + ) as t + inner join partsupp_1 + on t.l_partkey = partsupp_1.ps_partkey and t.l_suppkey = partsupp_1.ps_suppkey + where partsupp_1.ps_suppkey > 1 + group by l_orderkey, l_partkey, l_suppkey, o_orderkey, o_custkey, ps_partkey, ps_suppkey, agg1, agg3, agg4, agg5, agg6 ) as t2 on t1.l_orderkey = t2.l_orderkey + where t1.l_orderkey > 1 group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 ) as t2 on t1.l_orderkey = t2.l_orderkey group by t1.l_orderkey, t2.l_partkey, t1.l_suppkey, t2.o_orderkey, t1.o_custkey, t2.ps_partkey, t1.ps_suppkey, t2.agg1, t1.agg2, t2.agg3, t1.agg4, t2.agg5, t1.agg6 """ - explain { - sql("${sql_2}") - contains "${mv_2}(${mv_2})" - } + mv_rewrite_any_success(sql_2, [mv_1, mv_2]) compare_res(sql_2 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") - // tmp and will fix soon -// explain { -// sql("${sql_3}") -// contains "${mv_3}(${mv_3})" -// } -// compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") -// -// explain { -// sql("${sql_4}") -// contains "${mv_4}(${mv_4})" -// } -// compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") -// -// explain { -// sql("${sql_5}") -// contains "${mv_5}(${mv_5})" -// } -// compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") + mv_rewrite_any_success(sql_3, [mv_3, mv_4]) + compare_res(sql_3 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") + + mv_rewrite_any_success(sql_4, [mv_3, mv_4]) + compare_res(sql_4 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") + + mv_rewrite_any_success(sql_5, [mv_3, mv_4, mv_5]) + compare_res(sql_5 + " order by 1,2,3,4,5,6,7,8,9,10,11,12,13") } diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy index f9a84fa5250179..64354267a039bd 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy @@ -99,6 +99,9 @@ suite("nested_mtmv_rewrite_switch") { sql """analyze table orders_2 with sync;""" sql """analyze table lineitem_2 with sync;""" + sql """alter table orders_2 modify column o_orderdate set stats ('row_count'='10');""" + sql """alter table lineitem_2 modify column l_shipdate set stats ('row_count'='7');""" + def compare_res = { def stmt -> sql "SET enable_materialized_view_rewrite=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy index 35a983c264c0e2..f8023a9e1e4c49 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy @@ -480,6 +480,10 @@ suite("partition_mv_rewrite") { analyze table orders with sync; """ + sql """alter table orders modify column o_comment set stats ('row_count'='3');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='6');""" + sql """alter table lineitem_static modify column l_comment set stats ('row_count'='4');""" + // should rewrite successful when union rewrite enalbe if base table add new partition mv_rewrite_success(roll_up_all_partition_sql, "mv_10086", true, is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) diff --git a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy index 6b5c2dc45e875f..c42888e35abc0a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy @@ -123,6 +123,10 @@ suite("mv_scan_table") { sql """analyze table lineitem with sync;""" sql """analyze table partsupp with sync;""" + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" +sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + // with filter def mv1_0 = """ diff --git a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy index dfbe433fb9b3b0..24122f18da52e4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_all_compensate/union_all_compensate.groovy @@ -179,6 +179,9 @@ suite("union_all_compensate") { mv_rewrite_fail(query1_0, "test_agg_mv") order_qt_query1_1_after "${query1_0}" + sql """alter table test_table1 modify column num set stats ('row_count'='20');""" + sql """alter table test_table2 modify column num set stats ('row_count'='16');""" + // Aggregate, if query group by expression doesn't use the partition column, but the invalid partition is in the // grace_period, should not compensate union all, but should rewritten successfully diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy index 2b00c8200d2589..d1bdba31794acb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy @@ -78,6 +78,9 @@ suite ("partition_curd_union_rewrite") { ); """ + sql """alter table orders modify column o_comment set stats ('row_count'='3');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='3');""" + sql""" insert into orders values (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'), diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy index 702431e5e391ae..4e74090b1a2bb2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy @@ -90,6 +90,9 @@ suite ("usercase_union_rewrite") { sql """analyze table orders_user with sync;""" sql """analyze table lineitem_user with sync;""" + sql """alter table orders_user modify column o_comment set stats ('row_count'='4');""" + sql """alter table lineitem_user modify column l_comment set stats ('row_count'='3');""" + def create_mv_orders = { mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name};""" sql """DROP TABLE IF EXISTS ${mv_name}""" diff --git a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy index 5c64379950f942..cba2509483b6dc 100644 --- a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy @@ -57,6 +57,8 @@ suite("null_unsafe_equals") { (5, 2, 'o', 1.2, '2023-12-12', 'c','d', null, 'mi'); """ + sql """alter table orders modify column o_comment set stats ('row_count'='8');""" + def mv1_0 = """ select count(*), o_orderstatus, o_comment diff --git a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy index 62da95cf8c0149..9dddde1098f684 100644 --- a/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/variant/variant_mv.groovy @@ -87,6 +87,9 @@ suite("variant_mv") { sql """analyze table github_events1 with sync;""" sql """analyze table github_events2 with sync;""" + sql """alter table github_events1 modify column created_at set stats ('row_count'='3');""" + sql """alter table github_events2 modify column created_at set stats ('row_count'='3');""" + // variant appear in where both slot and in expression def mv1_0 = """ SELECT diff --git a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy index 400e113b6f3ae5..c51dea2f5a7403 100644 --- a/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/with_auth/with_select_table_auth.groovy @@ -112,6 +112,9 @@ suite("with_select_table_auth","p0,auth") { sql """analyze table lineitem with sync""" sql """analyze table orders with sync""" + sql """alter table orders modify column o_comment set stats ('row_count'='18');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """grant select_priv on ${db}.orders to ${user_name}""" sql """grant select_priv on ${db}.lineitem to ${user_name}""" sql """grant select_priv on regression_test to ${user_name}""" diff --git a/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy b/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy index b7c6ecbd8ae008..757a4c42e5ad32 100644 --- a/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/with_sql_limit/query_with_sql_limit.groovy @@ -143,6 +143,10 @@ suite("query_with_sql_limit") { sql """analyze table lineitem with sync""" sql """analyze table orders with sync""" + sql """alter table orders modify column o_comment set stats ('row_count'='18');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='3');""" + // test sql_select_limit default, default 9223372036854775807 sql """set sql_select_limit = 2;""" def mv1_0 = diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy index 4373e0b7073284..5cbdf2994e5a5f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/aggHaveDupBase.groovy @@ -36,6 +36,7 @@ suite ("agg_have_dup_base") { distributed BY hash(k1) buckets 3 properties("replication_num" = "1"); """ + sql """alter table agg_have_dup_base modify column k1 set stats ('row_count'='5');""" sql "insert into agg_have_dup_base select 1,1,1,'a';" sql "insert into agg_have_dup_base select 2,2,2,'b';" diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy index 91293bb747aa64..e2f334566e9c24 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/case_ignore.groovy @@ -36,6 +36,7 @@ suite ("case_ignore") { sql "insert into case_ignore select 2,2,2,'b';" sql "insert into case_ignore select 3,-3,null,'c';" + createMV ("create materialized view k12a as select K1,abs(K2) from case_ignore;") sleep(3000) @@ -55,6 +56,7 @@ suite ("case_ignore") { order_qt_select_mv "select K1,abs(K2) from case_ignore order by K1;" sql """set enable_stats=true;""" + sql """alter table case_ignore modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,abs(k2) from case_ignore order by k1;", "k12a") mv_rewrite_success("select K1,abs(K2) from case_ignore order by K1;", "k12a") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy index 132a4c3f054571..3537777d312c58 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_abs.groovy @@ -57,6 +57,7 @@ suite ("dup_gb_mv_abs") { order_qt_select_mv_sub "select sum(abs(k2)) from dup_gb_mv_abs group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_gb_mv_abs modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(abs(k2)) from dup_gb_mv_abs group by k1;", "k12sa") mv_rewrite_success("select sum(abs(k2)) from dup_gb_mv_abs group by k1;", "k12sa") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy index 45271415b98ef5..2e6e9c6b627850 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_gb_mv_plus.groovy @@ -57,6 +57,7 @@ suite ("dup_gb_mv_plus") { order_qt_select_mv_sub "select sum(k2+1) from dup_gb_mv_plus group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_gb_mv_plus modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,sum(k2+1) from dup_gb_mv_plus group by k1;", "k12sp") mv_rewrite_success("select sum(k2+1) from dup_gb_mv_plus group by k1;", "k12sp") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy index c4770d6dcd0262..6cef52fd96edc7 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_abs.groovy @@ -31,7 +31,6 @@ suite ("dup_mv_abs") { distributed BY hash(k1) buckets 3 properties("replication_num" = "1"); """ - sql "insert into dup_mv_abs select 1,1,1,'a';" sql "insert into dup_mv_abs select 2,2,2,'b';" sql "insert into dup_mv_abs select 3,-3,null,'c';" @@ -69,6 +68,7 @@ suite ("dup_mv_abs") { order_qt_select_group_mv_not "select sum(abs(k2)) from dup_mv_abs group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table dup_mv_abs modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,abs(k2) from dup_mv_abs order by k1;", "k12a") mv_rewrite_success("select abs(k2) from dup_mv_abs order by k1;", "k12a") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy index 788647ae224d98..7fd22f5b71892c 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bin.groovy @@ -69,6 +69,7 @@ suite ("dup_mv_bin") { order_qt_select_group_mv_not "select group_concat(bin(k2)) from dup_mv_bin group by k3 order by k3;" sql """set enable_stats=true;""" + sql """alter table dup_mv_bin modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,bin(k2) from dup_mv_bin order by k1;", "k12b") mv_rewrite_success("select bin(k2) from dup_mv_bin order by k1;", "k12b") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy index d049a0aeb7c1b0..f5784e7a5fdaa6 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_bm_hash.groovy @@ -45,6 +45,7 @@ suite ("dup_mv_bm_hash") { order_qt_select_mv "select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_mv_bm_hash modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select bitmap_union_count(to_bitmap(k2)) from dup_mv_bm_hash group by k1 order by k1;", "dup_mv_bm_hash_mv1") createMV("create materialized view dup_mv_bm_hash_mv2 as select k1,bitmap_union(bitmap_hash(k3)) from dup_mv_bm_hash group by k1;") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy index 0b7dce804b2504..77a12a3768ae87 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_plus.groovy @@ -46,7 +46,6 @@ suite ("dup_mv_plus") { sql "analyze table dup_mv_plus with sync;" sql """set enable_stats=false;""" - order_qt_select_star "select * from dup_mv_plus order by k1;" mv_rewrite_success("select k1,k2+1 from dup_mv_plus order by k1;", "k12p") @@ -87,6 +86,7 @@ suite ("dup_mv_plus") { order_qt_select_mv "select k1,k2+1 from dup_mv_plus order by k2;" sql """set enable_stats=true;""" + sql """alter table dup_mv_plus modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,k2+1 from dup_mv_plus order by k1;", "k12p") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy index c5098c55074c7d..95902c27866481 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/dup_mv_year.groovy @@ -45,6 +45,7 @@ suite ("dup_mv_year") { order_qt_select_mv "select k1,year(k2) from dup_mv_year order by k1;" sql """set enable_stats=true;""" + sql """alter table dup_mv_year modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1,year(k2) from dup_mv_year order by k1;", "k12y") createMV "create materialized view k13y as select k1,year(k3) from dup_mv_year;" diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy index 7e317d501bea07..d0fa214e98f042 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot1.groovy @@ -52,5 +52,6 @@ suite ("multi_slot1") { order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3;" sql """set enable_stats=true;""" + sql """alter table multi_slot1 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot1 order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy index 6bbded534a77ee..0fa36b77b44114 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot2.groovy @@ -65,6 +65,7 @@ suite ("multi_slot2") { order_qt_select_base "select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2 order by abs(k1)+k2;" sql """set enable_stats=true;""" + sql """alter table multi_slot2 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,sum(abs(k2+2)+k3+3) from multi_slot2 group by abs(k1)+k2+1 order by abs(k1)+k2+1", "k1a2p2ap3ps") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy index c6016296e8b352..51cc90aaf6ae26 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot3.groovy @@ -53,5 +53,6 @@ suite ("multi_slot3") { order_qt_select_mv "select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;" sql """set enable_stats=true;""" + sql """alter table multi_slot3 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select k1+1,abs(k2+2)+k3+3 from multi_slot3 order by k1+1;", "k1p2ap3p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy index f8210f914060f7..1112b93d0a6d2f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot4.groovy @@ -55,5 +55,6 @@ suite ("multi_slot4") { order_qt_select_mv "select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;" sql """set enable_stats=true;""" + sql """alter table multi_slot4 modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1+1,sum(abs(k2+2)+k3+3) from multi_slot4 group by k1+1 order by k1+1;", "k1p2ap3ps") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy index e10bed2dced7ef..7fa2e7ba6cda6f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot5.groovy @@ -63,5 +63,6 @@ suite ("multi_slot5") { order_qt_select_mv "select k1,version() from multi_slot5 order by k1;" sql """set enable_stats=true;""" + sql """alter table multi_slot5 modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k2+k3 from multi_slot5 order by k1;", "k123p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy index 9201a1cea4d321..2ed66fc8477a37 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/multi_slot6.groovy @@ -84,5 +84,6 @@ suite ("multi_slot6") { order_qt_select_mv "select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3;" sql """set enable_stats=true;""" + sql """alter table multi_slot6 modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+k2+1,abs(k2+2)+k3+3 from multi_slot6 order by abs(k1)+k2+1,abs(k2+2)+k3+3", "k1a2p2ap3p") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy index d6cd0d5a24421a..387c155a5e45c8 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/single_slot.groovy @@ -47,7 +47,6 @@ suite ("single_slot") { sql "analyze table single_slot with sync;" sql """set enable_stats=false;""" - order_qt_select_star "select * from single_slot order by k1;" explain { @@ -58,5 +57,6 @@ suite ("single_slot") { order_qt_select_mv "select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;" sql """set enable_stats=true;""" + sql """alter table single_slot modify column k1 set stats ('row_count'='4');""" mv_rewrite_success("select abs(k1)+1 t,sum(abs(k2+1)) from single_slot group by t order by t;", "k1ap2spa") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy index 031560a41ef502..be8bc50532ff81 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/sum_devide_count.groovy @@ -62,7 +62,7 @@ suite ("sum_devide_count") { order_qt_select_mv "select sum(k2)/count(k2) from sum_devide_count;" sql """set enable_stats=true;""" - + sql """alter table sum_devide_count modify column k1 set stats ('row_count'='5');""" mv_rewrite_success("select k1,k4,sum(k2)/count(k2) from sum_devide_count group by k1,k4 order by k1,k4;", "kavg") mv_rewrite_success("select k1,sum(k2)/count(k2) from sum_devide_count group by k1 order by k1;", "kavg") diff --git a/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy b/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy index df390b26551054..1ebaa8884ab383 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/newMv/unique_mv.groovy @@ -47,6 +47,7 @@ suite ("unique_mv") { mv_rewrite_success("SELECT * FROM c5816_t WHERE call_uuid='adc';", "mv_1") sql """set enable_stats=true;""" + sql """alter table c5816_t modify column org_id set stats ('row_count'='1');""" mv_rewrite_success("SELECT * FROM c5816_t WHERE call_uuid='adc';", "mv_1") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy index 9a5e42034be3cb..2b6b0dd354e4cd 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVMultiUsage.groovy @@ -37,6 +37,7 @@ suite ("MVMultiUsage") { sql """insert into MVMultiUsage values("2020-01-02",2,"b",2,2,2);""" sql """insert into MVMultiUsage values("2020-01-03",3,"c",3,3,3);""" + createMV("create materialized view MVMultiUsage_mv as select deptno, empid, salary from MVMultiUsage order by deptno;") sleep(3000) @@ -58,6 +59,8 @@ suite ("MVMultiUsage") { order_qt_select_mv "select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno) order by 1;" sql """set enable_stats=true;""" + sql """alter table MVMultiUsage modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from MVMultiUsage order by empid;", "MVMultiUsage_mv") explain { sql("select * from (select deptno, empid from MVMultiUsage where deptno>100) A join (select deptno, empid from MVMultiUsage where deptno >200) B using (deptno);") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy index 246c97022e9e85..86a85dcc647443 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/MVWithAs.groovy @@ -30,6 +30,7 @@ suite ("MVWithAs") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" sql """insert into MVWithAs values("2020-01-01",1,"a",1);""" @@ -53,6 +54,8 @@ suite ("MVWithAs") { order_qt_select_mv "select count(tag_id) from MVWithAs t;" sql """set enable_stats=true;""" + sql """alter table MVWithAs modify column time_col set stats ('row_count'='7');""" + mv_rewrite_fail("select * from MVWithAs order by time_col;", "MVWithAs_mv") mv_rewrite_success("select count(tag_id) from MVWithAs t;", "MVWithAs_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy index 9500c6ff1919ae..b48db921cc9a5d 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggCDInBitmap.groovy @@ -31,6 +31,8 @@ suite ("aggCDInBitmap") { sql """insert into aggCDInBitmap values(2,to_bitmap(2));""" sql """insert into aggCDInBitmap values(3,to_bitmap(3));""" + sql "analyze table aggCDInBitmap with sync;" + sql """set enable_stats=false;""" order_qt_select_star "select * from aggCDInBitmap order by 1;" @@ -41,4 +43,11 @@ suite ("aggCDInBitmap") { } order_qt_select_mv "select k1, count(distinct v1) from aggCDInBitmap group by k1 order by k1;" + sql """set enable_stats=true;""" + sql """alter table aggCDInBitmap modify column k1 set stats ('row_count'='3');""" + explain { + sql("select k1, count(distinct v1) from aggCDInBitmap group by k1;") + contains "bitmap_union_count" + } + } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy index 1a7ccd851503d8..a234661150e732 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggMVCalcAggFun.groovy @@ -38,7 +38,6 @@ suite ("aggMVCalcAggFun") { sql """insert into aggMVCalcAggFun values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggMVCalcAggFun values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view aggMVCalcAggFunMv as select deptno, empid, sum(salary) from aggMVCalcAggFun group by empid, deptno;") sleep(3000) @@ -55,6 +54,7 @@ suite ("aggMVCalcAggFun") { order_qt_select_mv "select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table aggMVCalcAggFun modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggMVCalcAggFun order by empid;", "aggMVCalcAggFunMv")\ mv_rewrite_fail("select deptno, sum(salary + 1) from aggMVCalcAggFun where deptno > 10 group by deptno;", "aggMVCalcAggFunMv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy index 0cb12cbcde0b51..c29eec40c8da93 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV1.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV1") { order_qt_select_mv "select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV1 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV1 order by empid;", "aggOnAggMV1_mv") mv_rewrite_success("select sum(salary), deptno from aggOnAggMV1 group by deptno order by deptno;", "aggOnAggMV1_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy index bd826537ff88f7..e38a27c1db3c53 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV10.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV10") { order_qt_select_mv "select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission) order by 1,2;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV10 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV10 order by empid;", "aggOnAggMV10_mv") mv_rewrite_success("select deptno, commission, sum(salary) + 1 from aggOnAggMV10 group by rollup (deptno, commission);", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy index 793389ec68e4ae..b0f379b19476b8 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV11.groovy @@ -33,6 +33,7 @@ suite ("aggOnAggMV11") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into aggOnAggMV11 values("2020-01-01",1,"a",1,1,1);""" sql """insert into aggOnAggMV11 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV11 values("2020-01-03",3,"c",3,3,3);""" @@ -54,6 +55,8 @@ suite ("aggOnAggMV11") { order_qt_select_mv "select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV11 modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from aggOnAggMV11 order by empid;", "aggOnAggMV11_mv") mv_rewrite_fail("select deptno, count(salary) + count(1) from aggOnAggMV11 group by deptno;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy index 32d897939ea652..179e22584d32a1 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV2.groovy @@ -56,6 +56,7 @@ suite ("aggOnAggMV2") { order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV2 modify column time_col set stats ('row_count'='3');""" mv_rewrite_fail("select * from aggOnAggMV2 order by empid;", "aggOnAggMV2_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV2 group by deptno) a where (sum_salary * 2) > 3 order by deptno ;", "aggOnAggMV2_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy index e03c6d8b195a68..6816e163261986 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV3.groovy @@ -55,6 +55,7 @@ suite ("aggOnAggMV3") { order_qt_select_mv "select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV3 modify column time_col set stats ('row_count'='5');""" mv_rewrite_fail("select * from aggOnAggMV3 order by empid;", "aggOnAggMV3_mv") mv_rewrite_success("select commission, sum(salary) from aggOnAggMV3 where commission * (deptno + commission) = 100 group by commission order by commission;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy index 481d8f19b5a133..d90057b8bf5120 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV5.groovy @@ -33,6 +33,8 @@ suite ("aggOnAggMV5") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table aggOnAggMV5 modify column time_col set stats ('row_count'='4');""" + sql """insert into aggOnAggMV5 values("2020-01-01",1,"a",1,1,1);""" sql """insert into aggOnAggMV5 values("2020-01-02",2,"b",2,2,2);""" sql """insert into aggOnAggMV5 values("2020-01-03",3,"c",3,3,3);""" diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy index abd66e1ab13ccd..7e2e87718b8e76 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV6.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV6") { order_qt_select_mv "select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10 order by 1;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV6 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV6 order by empid;", "aggOnAggMV6_mv") mv_rewrite_success("select * from (select deptno, sum(salary) as sum_salary from aggOnAggMV6 where deptno>=20 group by deptno) a where sum_salary>10;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy index 53b75d03d85167..386871a8a5a98f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/aggOnAggMV7.groovy @@ -54,6 +54,7 @@ suite ("aggOnAggMV7") { order_qt_select_mv "select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno order by 1;" sql """set enable_stats=true;""" + sql """alter table aggOnAggMV7 modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from aggOnAggMV7 order by empid;", "aggOnAggMV7_mv") mv_rewrite_success("select deptno, sum(salary) from aggOnAggMV7 where deptno>=20 group by deptno;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy index 13e2097cdb46ba..8b2697cb1f78f7 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/bitmapUnionIn.groovy @@ -30,6 +30,7 @@ suite ("bitmapUnionIn") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into bitmapUnionIn values("2020-01-01",1,"a",1);""" sql """insert into bitmapUnionIn values("2020-01-02",2,"b",2);""" @@ -50,6 +51,8 @@ suite ("bitmapUnionIn") { order_qt_select_mv "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;" sql """set enable_stats=true;""" + sql """alter table bitmapUnionIn modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from bitmapUnionIn order by time_col;", "bitmapUnionIn_mv") mv_rewrite_success("select user_id, bitmap_union_count(to_bitmap(tag_id)) a from bitmapUnionIn group by user_id having a>1 order by a;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy index c28dac1d227aab..52abff46678bb0 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incMVReInSub.groovy @@ -31,6 +31,7 @@ suite ("incMVReInSub") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into incMVReInSub values("2020-01-01",1,"a",1);""" sql """insert into incMVReInSub values("2020-01-02",2,"b",2);""" @@ -52,6 +53,8 @@ suite ("incMVReInSub") { order_qt_select_mv "select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;" sql """set enable_stats=true;""" + sql """alter table incMVReInSub modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from incMVReInSub order by time_col;", "incMVReInSub_mv") mv_rewrite_fail("select user_id, bitmap_union(to_bitmap(tag_id)) from incMVReInSub where user_name in (select user_name from incMVReInSub group by user_name having bitmap_union_count(to_bitmap(tag_id)) >1 ) group by user_id order by user_id;", diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy index ee8e688c0e7313..6172773ca7a43b 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/incRewriteCD.groovy @@ -31,6 +31,7 @@ suite ("incRewriteCD") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into incRewriteCD values("2020-01-01",1,"a",1);""" sql """insert into incRewriteCD values("2020-01-02",2,"b",2);""" @@ -50,6 +51,8 @@ suite ("incRewriteCD") { order_qt_select_mv "select user_name, count(distinct tag_id) from incRewriteCD group by user_name order by user_name;" sql """set enable_stats=true;""" + sql """alter table incRewriteCD modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from incRewriteCD order by time_col;", "incRewriteCD_mv") mv_rewrite_fail("select user_name, count(distinct tag_id) from incRewriteCD group by user_name;", "incRewriteCD_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy index f2f7a014d98fc8..69d1c39931c17f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/joinOnLeftPToJoin.groovy @@ -31,6 +31,8 @@ suite ("joinOnLeftPToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """alter table joinOnLeftPToJoin modify column time_col set stats ('row_count'='3');""" + sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,2,2);""" sql """insert into joinOnLeftPToJoin values("2020-01-03",3,"c",3,3,3);""" sql """insert into joinOnLeftPToJoin values("2020-01-02",2,"b",2,7,2);""" @@ -45,6 +47,7 @@ suite ("joinOnLeftPToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",2);""" sql """insert into joinOnLeftPToJoin_1 values("2020-01-03",3,"c",3);""" sql """insert into joinOnLeftPToJoin_1 values("2020-01-02",2,"b",1);""" @@ -64,6 +67,8 @@ suite ("joinOnLeftPToJoin") { order_qt_select_mv "select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno order by A.deptno;" sql """set enable_stats=true;""" + sql """alter table joinOnLeftPToJoin_1 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_all_success("select * from (select deptno , sum(salary) from joinOnLeftPToJoin group by deptno) A join (select deptno, max(cost) from joinOnLeftPToJoin_1 group by deptno ) B on A.deptno = B.deptno;", ["joinOnLeftPToJoin_mv", "joinOnLeftPToJoin_1_mv"]) } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy index 7f455da429e6c3..21864f973ee8c0 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/orderByOnPView.groovy @@ -54,6 +54,8 @@ suite ("orderByOnPView") { order_qt_select_mv "select empid from orderByOnPView order by deptno;" sql """set enable_stats=true;""" + sql """alter table orderByOnPView modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from orderByOnPView where time_col='2020-01-01' order by empid;", "orderByOnPView_mv") mv_rewrite_success("select empid from orderByOnPView where deptno = 0 order by deptno;", "orderByOnPView_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy index 384137471d827f..157a15498e717c 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV1.groovy @@ -33,6 +33,7 @@ suite ("projectMV1") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV1 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV1 values("2020-01-02",2,"b",2,2,2);""" @@ -52,6 +53,8 @@ suite ("projectMV1") { order_qt_select_mv "select empid, deptno from projectMV1 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV1 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV1 where time_col='2020-01-01' order by empid;", "projectMV1_mv") mv_rewrite_success("select empid, deptno from projectMV1 where deptno=0 order by empid;", "projectMV1_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy index 672bda198b01aa..4d85fcc9c8aeae 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV2.groovy @@ -33,6 +33,7 @@ suite ("projectMV2") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV2 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV2 values("2020-01-02",2,"b",2,2,2);""" @@ -55,6 +56,8 @@ suite ("projectMV2") { order_qt_select_base "select name from projectMV2 where deptno -1 = 0 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV2 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV2 order by empid;", "projectMV2_mv") mv_rewrite_success("select empid + 1 from projectMV2 where deptno = 1 order by empid;", "projectMV2_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy index 37b73a9a81aff4..9ad301fbbc6a97 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV3.groovy @@ -33,6 +33,7 @@ suite ("projectMV3") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV3 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV3 values("2020-01-02",2,"b",2,2,2);""" @@ -57,6 +58,8 @@ suite ("projectMV3") { order_qt_select_mv2 "select name from projectMV3 where deptno -1 = 0 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV3 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV3 order by empid;", "projectMV3_mv") mv_rewrite_success("select empid + 1, name from projectMV3 where deptno = 1 order by empid;", "projectMV3_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy index ea87551dfa1249..ddcdada549ad8f 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/projectMV4.groovy @@ -33,6 +33,7 @@ suite ("projectMV4") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ + sql """insert into projectMV4 values("2020-01-01",1,"a",1,1,1);""" sql """insert into projectMV4 values("2020-01-02",2,"b",2,2,2);""" @@ -57,6 +58,8 @@ suite ("projectMV4") { order_qt_select_base "select empid from projectMV4 where deptno > 1 and empid > 1 order by empid;" sql """set enable_stats=true;""" + sql """alter table projectMV4 modify column time_col set stats ('row_count'='3');""" + mv_rewrite_fail("select * from projectMV4 order by empid;", "projectMV4_mv") mv_rewrite_success("select name from projectMV4 where deptno > 1 and salary > 1 and name = 'a' order by name;", "projectMV4_mv") diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy index 25300c9d602681..67312eed088ed4 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/subQuery.groovy @@ -37,7 +37,6 @@ suite ("subQuery") { sql """insert into subQuery values("2020-01-02",2,"b",2,2,2);""" sql """insert into subQuery values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view subQuery_mv as select deptno, empid from subQuery;") sleep(3000) @@ -60,6 +59,7 @@ suite ("subQuery") { */ sql """set enable_stats=true;""" + sql """alter table subQuery modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from subQuery order by empid;", "subQuery_mv") } diff --git a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy index 0f57b5c6c1687e..9feb1aa1c975f5 100644 --- a/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy +++ b/regression-test/suites/nereids_syntax_p0/mv/ut/unionDis.groovy @@ -58,6 +58,8 @@ suite ("unionDis") { order_qt_select_mv "select * from (select empid, deptno from unionDis where empid >1 union select empid, deptno from unionDis where empid <0) t order by 1;" sql """set enable_stats=true;""" + sql """alter table unionDis modify column time_col set stats ('row_count'='4');""" + mv_rewrite_fail("select * from unionDis order by empid;", "unionDis_mv") explain { diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy index ceada782a27788..ca429313770757 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/agg.groovy @@ -41,6 +41,7 @@ suite("agg") { AGGREGATE KEY (siteid,citycode,username) DISTRIBUTED BY HASH(siteid) BUCKETS 5 properties("replication_num" = "1"); """ + sql """alter table test_rollup_agg1 modify column siteid set stats ('row_count'='3');""" sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_city(citycode, pv);""" int max_try_secs = 60 while (max_try_secs--) { diff --git a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy index b711728f769129..07af0da35098e1 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/agg_date.groovy @@ -44,6 +44,7 @@ suite("agg_date", "rollup") { AGGREGATE KEY (datek1, datetimek1, datetimek2, datetimek3) DISTRIBUTED BY HASH(datek1) BUCKETS 5 properties("replication_num" = "1"); """ + sql """alter table test_rollup_agg_date1 modify column datek1 set stats ('row_count'='2');""" sql """ALTER TABLE ${tbName} ADD ROLLUP rollup_date(datek1,datetimek2,datetimek1,datetimek3,datev1,datetimev1,datetimev2,datetimev3);""" int max_try_secs = 60 while (max_try_secs--) { diff --git a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy index a3917dee28a75a..2407c70d8a451c 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/bitmap.groovy @@ -34,6 +34,7 @@ suite("bitmap", "rollup") { ) DISTRIBUTED BY HASH(k1) properties("replication_num" = "1"); """ + sql """alter table test_materialized_view_bitmap1 modify column k1 set stats ('row_count'='2');""" sql "CREATE MATERIALIZED VIEW test_neg as select k1,bitmap_union(to_bitmap(k2)), bitmap_union(to_bitmap(k3)) FROM ${tbName1} GROUP BY k1;" max_try_secs = 60 diff --git a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy index e1eb0ca34418e1..34d5e338cfa406 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/date.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/date.groovy @@ -126,6 +126,7 @@ suite("date", "rollup") { mv_rewrite_success("SELECT store_id, max(sale_datetime3) FROM ${tbName1} GROUP BY store_id", "amt_max4") sql """set enable_stats=true;""" + sql """alter table test_materialized_view_date1 modify column record_id set stats ('row_count'='2');""" mv_rewrite_success("SELECT store_id, max(sale_date1) FROM ${tbName1} GROUP BY store_id", "amt_max1") mv_rewrite_success("SELECT store_id, max(sale_datetime1) FROM ${tbName1} GROUP BY store_id", "amt_max2") diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy index 2ba3540832945a..5002c509a20617 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/hll/hll.groovy @@ -46,6 +46,7 @@ suite("hll", "rollup") { "amt_count") sql """set enable_stats=true;""" + sql """alter table test_materialized_view_hll1 modify column record_id set stats ('row_count'='2');""" mv_rewrite_success("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM test_materialized_view_hll1 GROUP BY store_id;", "amt_count") } diff --git a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy index 51599f294d1e62..d7dec7696d95eb 100644 --- a/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy +++ b/regression-test/suites/nereids_syntax_p0/rollup/hll_with_light_sc/hll_with_light_sc.groovy @@ -43,6 +43,7 @@ suite("hll_with_light_sc", "rollup") { "amt_count1") sql """set enable_stats=true;""" + sql """alter table test_materialized_view_hll_with_light_sc1 modify column record_id set stats ('row_count'='2');""" mv_rewrite_success("SELECT store_id, hll_union_agg(hll_hash(sale_amt)) FROM test_materialized_view_hll_with_light_sc1 GROUP BY store_id;", "amt_count1") } From 860dc07652db6f0265cfb4a10fac90a2230716bf Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 5 Dec 2024 23:13:19 +0800 Subject: [PATCH 4/7] [fix](mtmv) Fix rewrite fail by materialized view when filter or join condition has alias (#44779) Related PR: #27922 Problem Summary: query and mv def are as following,` partsupp.public_col as public_col ` is alias, this would cause rewritting fail by materialized view with msg, the graph logic between query and view is different. select o_custkey, o_orderdate, o_shippriority, o_comment, o_orderkey, orders.public_col as col1, l_orderkey, l_partkey, l_suppkey, lineitem.public_col as col2, ps_partkey, ps_suppkey, partsupp.public_col as col3, partsupp.public_col * 2 as col4, o_orderkey + l_orderkey + ps_partkey * 2, sum( o_orderkey + l_orderkey + ps_partkey * 2 ), count() as count_all from ( select o_custkey, o_orderdate, o_shippriority, o_comment, o_orderkey, orders.public_col as public_col from orders ) orders left join ( select l_orderkey, l_partkey, l_suppkey, lineitem.public_col as public_col from lineitem where lineitem.public_col is null or lineitem.public_col <> 1 ) lineitem on l_orderkey = o_orderkey inner join ( select ps_partkey, ps_suppkey, partsupp.public_col as public_col from partsupp ) partsupp on ps_partkey = o_orderkey where lineitem.public_col is null or lineitem.public_col <> 1 and o_orderkey = 2 group by 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14; Fix rewrite fail by materialized view when filter or join condition has alias --- .../joinorder/hypergraph/HyperElement.java | 27 + .../jobs/joinorder/hypergraph/edge/Edge.java | 4 +- .../hypergraph/node/AbstractNode.java | 8 +- .../exploration/mv/HyperGraphComparator.java | 103 +++- .../mv/LogicalCompatibilityContext.java | 78 ++- .../rules/exploration/mv/StructInfo.java | 136 +++-- .../aggregate_without_roll_up.out | 24 + .../aggregate_without_roll_up.groovy | 463 +++++++++++++++++- 8 files changed, 744 insertions(+), 99 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java new file mode 100644 index 00000000000000..6d8d7c6326c0d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperElement.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.joinorder.hypergraph; + +/** + * This is the common base class for all + * */ +public interface HyperElement { + + // Get the references nodes + long getReferenceNodes(); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java index 81a60b25031ca2..c9979ad002f702 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.jobs.joinorder.hypergraph.edge; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -32,7 +33,7 @@ /** * Edge in HyperGraph */ -public abstract class Edge { +public abstract class Edge implements HyperElement { private final int index; private final double selectivity; @@ -192,6 +193,7 @@ public boolean isSub(Edge edge) { return LongBitmap.isSubset(getReferenceNodes(), otherBitmap); } + @Override public long getReferenceNodes() { return LongBitmap.newBitmapUnion(leftExtendedNodes, rightExtendedNodes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java index a4a64e0449deee..686576de771d94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/node/AbstractNode.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.jobs.joinorder.hypergraph.node; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge; @@ -33,7 +34,7 @@ /** * HyperGraph Node. */ -public class AbstractNode { +public class AbstractNode implements HyperElement { protected final int index; protected final List joinEdges; protected final List filterEdges; @@ -65,6 +66,11 @@ public List getEdges() { .build(); } + @Override + public long getReferenceNodes() { + return getNodeMap(); + } + public int getIndex() { return index; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index e754e2f41281a9..ff1a5bdd09610b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -19,12 +19,14 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.jobs.joinorder.hypergraph.ConflictRulesMaker; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -51,6 +53,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -79,9 +82,9 @@ public class HyperGraphComparator { private final Map> pullUpViewExprWithEdge = new HashMap<>(); private final LogicalCompatibilityContext logicalCompatibilityContext; // this records the slots which needs to reject null - // the key is the target join which should reject null, the value is a pair, the first value of the pair is the - // join type, the second value is also a pair which left represents the slots in the left of join that should - // reject null, right represents the slots in the right of join that should reject null. + // the key is the view join edge which should reject null, the value is a pair, the first value of the pair is the + // query join type, the second value is also a pair which left represents the slots in the left of view join that + // should reject null, right represents the slots in the right of view join that should reject null. private final Map, Set>>> inferredViewEdgeWithCond = new HashMap<>(); private List viewJoinEdgesAfterInferring; private List viewFilterEdgesAfterInferring; @@ -249,9 +252,17 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) { } int size = queryExprSetList.size(); for (int i = 0; i < size; i++) { - Set mappingQueryExprSet = queryExprSetList.get(i).stream() - .map(logicalCompatibilityContext::getViewNodeExprFromQuery) - .collect(Collectors.toSet()); + Set queryExpressions = queryExprSetList.get(i); + Set mappingQueryExprSet = new HashSet<>(); + for (Expression queryExpression : queryExpressions) { + Optional mappingViewExprByQueryExpr = getMappingViewExprByQueryExpr(queryExpression, query, + this.logicalCompatibilityContext, + ExpressionPosition.NODE); + if (!mappingViewExprByQueryExpr.isPresent()) { + return false; + } + mappingQueryExprSet.add(mappingViewExprByQueryExpr.get()); + } if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) { return false; } @@ -407,7 +418,10 @@ private Map constructQueryToViewJoinMapWithExpr() { if (edgeMap.containsKey(entry.getValue())) { continue; } - Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey()); + Expression viewExpr = getMappingViewExprByQueryExpr(entry.getKey(), + entry.getValue(), + logicalCompatibilityContext, + ExpressionPosition.JOIN_EDGE).orElse(null); if (viewExprToEdge.containsKey(viewExpr)) { edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); } @@ -441,15 +455,19 @@ private Map constructQueryToViewFilterMapWithExpr() { HashMap queryToViewEdgeMap = new HashMap<>(); for (Entry> entry : queryExprToEdge.asMap().entrySet()) { - Expression queryExprViewBased = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey()); - if (queryExprViewBased == null) { - continue; - } - Collection viewEdges = viewExprToEdge.get(queryExprViewBased); - if (viewEdges.isEmpty()) { - continue; - } + Expression queryExprViewBased = null; for (Edge queryEdge : entry.getValue()) { + queryExprViewBased = getMappingViewExprByQueryExpr(entry.getKey(), + queryEdge, + logicalCompatibilityContext, + ExpressionPosition.FILTER_EDGE).orElse(null); + if (queryExprViewBased == null) { + continue; + } + Collection viewEdges = viewExprToEdge.get(queryExprViewBased); + if (viewEdges.isEmpty()) { + continue; + } for (Edge viewEdge : viewEdges) { if (!isSubTreeNodesEquals(queryEdge, viewEdge, logicalCompatibilityContext)) { // Such as query filter edge is <{1} --FILTER-- {}> but view filter edge is @@ -512,17 +530,17 @@ private boolean compareEdgeWithNode(Edge query, Edge view) { } private boolean compareFilterEdgeWithNode(FilterEdge query, FilterEdge view) { - return rewriteQueryNodeMap(query.getReferenceNodes()) == view.getReferenceNodes(); + return getViewNodesByQuery(query.getReferenceNodes()) == view.getReferenceNodes(); } private boolean compareJoinEdgeWithNode(JoinEdge query, JoinEdge view) { boolean res = false; if (query.getJoinType().swap() == view.getJoinType()) { - res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getRightExtendedNodes() - && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getLeftExtendedNodes(); + res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getRightExtendedNodes() + && getViewNodesByQuery(query.getRightExtendedNodes()) == view.getLeftExtendedNodes(); } - res |= rewriteQueryNodeMap(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes() - && rewriteQueryNodeMap(query.getRightExtendedNodes()) == view.getRightExtendedNodes(); + res |= getViewNodesByQuery(query.getLeftExtendedNodes()) == view.getLeftExtendedNodes() + && getViewNodesByQuery(query.getRightExtendedNodes()) == view.getRightExtendedNodes(); return res; } @@ -545,8 +563,8 @@ private boolean compareJoinEdgeOrInfer(JoinEdge query, JoinEdge view) { } private boolean tryInferEdge(JoinEdge query, JoinEdge view) { - if (rewriteQueryNodeMap(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes() - || rewriteQueryNodeMap(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) { + if (getViewNodesByQuery(query.getLeftRequiredNodes()) != view.getLeftRequiredNodes() + || getViewNodesByQuery(query.getRightRequiredNodes()) != view.getRightRequiredNodes()) { return false; } if (!query.getJoinType().equals(view.getJoinType())) { @@ -567,7 +585,7 @@ private boolean tryInferEdge(JoinEdge query, JoinEdge view) { return true; } - private long rewriteQueryNodeMap(long bitmap) { + private long getViewNodesByQuery(long bitmap) { long newBitmap = LongBitmap.newBitmap(); for (int i : LongBitmap.getIterator(bitmap)) { int newIdx = getQueryToViewNodeIdMap().getOrDefault(i, 0); @@ -576,6 +594,35 @@ private long rewriteQueryNodeMap(long bitmap) { return newBitmap; } + private Optional getMappingViewExprByQueryExpr(Expression queryExpression, + HyperElement queryExpressionBelongedHyperElement, + LogicalCompatibilityContext context, + ExpressionPosition expressionPosition) { + Expression queryShuttledExpr; + Collection> viewExpressions; + if (ExpressionPosition.JOIN_EDGE.equals(expressionPosition)) { + queryShuttledExpr = context.getQueryJoinShuttledExpr(queryExpression); + viewExpressions = context.getViewJoinExprFromQuery(queryShuttledExpr); + } else if (ExpressionPosition.FILTER_EDGE.equals(expressionPosition)) { + queryShuttledExpr = context.getQueryFilterShuttledExpr(queryExpression); + viewExpressions = context.getViewFilterExprFromQuery(queryShuttledExpr); + } else { + queryShuttledExpr = context.getQueryNodeShuttledExpr(queryExpression); + viewExpressions = context.getViewNodeExprFromQuery(queryShuttledExpr); + } + if (viewExpressions.size() == 1) { + return Optional.of(viewExpressions.iterator().next().key()); + } + long queryReferenceNodes = queryExpressionBelongedHyperElement.getReferenceNodes(); + long viewReferenceNodes = getViewNodesByQuery(queryReferenceNodes); + for (Pair viewExpressionPair : viewExpressions) { + if (viewExpressionPair.value().getReferenceNodes() == viewReferenceNodes) { + return Optional.of(viewExpressionPair.key()); + } + } + return Optional.empty(); + } + private void compareJoinEdgeWithExpr(Edge query, Edge view) { Set queryExprSet = query.getExpressionSet(); Set viewExprSet = view.getExpressionSet(); @@ -583,7 +630,10 @@ private void compareJoinEdgeWithExpr(Edge query, Edge view) { Set exprMappedOfView = new HashSet<>(); List residualQueryExpr = new ArrayList<>(); for (Expression queryExpr : queryExprSet) { - Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr); + Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr, + query, + logicalCompatibilityContext, + ExpressionPosition.JOIN_EDGE).orElse(null); if (viewExprSet.contains(viewExpr)) { exprMappedOfView.add(viewExpr); } else { @@ -602,7 +652,10 @@ private void compareFilterEdgeWithExpr(Edge query, Edge view) { Set exprMappedOfView = new HashSet<>(); List residualQueryExpr = new ArrayList<>(); for (Expression queryExpr : queryExprSet) { - Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr); + Expression viewExpr = getMappingViewExprByQueryExpr(queryExpr, + query, + logicalCompatibilityContext, + ExpressionPosition.FILTER_EDGE).orElse(null); if (viewExprSet.contains(viewExpr)) { exprMappedOfView.add(viewExpr); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 06098b591857ad..77ab37873d06b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -17,7 +17,10 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition; import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; @@ -27,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; @@ -34,8 +38,10 @@ import com.google.common.base.Suppliers; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; -import java.util.HashMap; +import java.util.Collection; import java.util.Map; import java.util.function.Supplier; @@ -45,9 +51,16 @@ public class LogicalCompatibilityContext { private final BiMap queryToViewNodeMapping; private final BiMap queryToViewNodeIDMapping; - private final Supplier> queryToViewJoinEdgeExpressionMappingSupplier; - private final Supplier> queryToViewNodeExpressionMappingSupplier; - private final Supplier> queryToViewFilterEdgeExpressionMappingSupplier; + private final ObjectId planNodeId; + private final Supplier>> + queryToViewJoinEdgeExpressionMappingSupplier; + private final Supplier> queryToQueryShuttledJoinExpressionMappingSupplier; + private final Supplier>> + queryToViewNodeExpressionMappingSupplier; + private final Supplier> queryToQueryShuttledNodeExpressionMappingSupplier; + private final Supplier>> + queryToViewFilterEdgeExpressionMappingSupplier; + private final Supplier> queryToQueryShuttledFilterExpressionMappingSupplier; /** * LogicalCompatibilityContext @@ -61,19 +74,31 @@ private LogicalCompatibilityContext(BiMap queryT queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE))); + this.queryToQueryShuttledJoinExpressionMappingSupplier = Suppliers.memoize( + () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.JOIN_EDGE)); + this.queryToViewNodeExpressionMappingSupplier = Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE))); + this.queryToQueryShuttledNodeExpressionMappingSupplier = Suppliers.memoize( + () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.NODE)); + this.queryToViewFilterEdgeExpressionMappingSupplier = Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE), viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE))); + this.queryToQueryShuttledFilterExpressionMappingSupplier = Suppliers.memoize( + () -> queryStructInfo.getExpressionToShuttledExpressionToMap().get(ExpressionPosition.FILTER_EDGE)); + this.queryToViewNodeMapping = queryToViewNodeMapping; this.queryToViewNodeIDMapping = HashBiMap.create(); queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); + + this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); } public BiMap getQueryToViewNodeMapping() { @@ -84,18 +109,30 @@ public BiMap getQueryToViewNodeIDMapping() { return queryToViewNodeIDMapping; } - public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) { + public Collection> getViewJoinExprFromQuery(Expression queryJoinExpr) { return queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) { + public Expression getQueryJoinShuttledExpr(Expression queryJoinExpr) { + return queryToQueryShuttledJoinExpressionMappingSupplier.get().get(queryJoinExpr); + } + + public Collection> getViewFilterExprFromQuery(Expression queryJoinExpr) { return queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) { + public Expression getQueryFilterShuttledExpr(Expression queryFilterExpr) { + return queryToQueryShuttledFilterExpressionMappingSupplier.get().get(queryFilterExpr); + } + + public Collection> getViewNodeExprFromQuery(Expression queryJoinExpr) { return queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr); } + public Expression getQueryNodeShuttledExpr(Expression queryNodeExpr) { + return queryToQueryShuttledNodeExpressionMappingSupplier.get().get(queryNodeExpr); + } + /** * Generate logical compatibility context, * this make expression mapping between query and view by relation and the slot in relation mapping @@ -126,24 +163,31 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, viewStructInfo); } - private static BiMap generateExpressionMapping( + /** + * The result is multimap + * the key is shuttled query expr + * the value is original view expr collection + * */ + private static Multimap> generateExpressionMapping( Map viewToQuerySlotMapping, - Map queryShuttledExprToExprMap, - Map viewShuttledExprToExprMap) { - final Map viewEdgeToConjunctsMapQueryBased = new HashMap<>(); - BiMap queryToViewEdgeMapping = HashBiMap.create(); + Multimap> queryShuttledExprToExprMap, + Multimap> viewShuttledExprToExprMap) { + Multimap> queryToViewEdgeMapping = HashMultimap.create(); if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) { return queryToViewEdgeMapping; } + final Multimap> viewShuttledExprToExprMapQueryBased = + HashMultimap.create(); viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { - viewEdgeToConjunctsMapQueryBased.put( + viewShuttledExprToExprMapQueryBased.put( orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr); }); - queryShuttledExprToExprMap.forEach((exprSet, edge) -> { - Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet)); - if (viewExpr != null) { - queryToViewEdgeMapping.put(edge, viewExpr); + queryShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { + Collection> viewExpressions = viewShuttledExprToExprMapQueryBased.get( + orderSlotAsc(shuttledExpr)); + if (viewExpressions != null) { + queryToViewEdgeMapping.putAll(shuttledExpr, viewExpressions); } }); return queryToViewEdgeMapping; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index e1802981274ac0..28983a04f0e028 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -23,10 +23,10 @@ import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.jobs.executor.Rewriter; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperElement; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; -import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.TableQueryOperatorChecker; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; @@ -63,12 +63,15 @@ import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -111,9 +114,23 @@ public class StructInfo { // split predicates is shuttled private SplitPredicate splitPredicate; private EquivalenceClass equivalenceClass; - // Key is the expression shuttled and the value is the origin expression + // For value of Map, the key is the position of expression + // the value is the expressions and the hyper element of expression pair + // Key of pair is the expression shuttled and the value is the origin expression and the hyper element it belonged + // Sometimes origin expressions are different and shuttled expression is same + // Such as origin expressions are l_partkey#0 > 1 and l_partkey#10 > 1 and shuttled expression is l_partkey#10 > 1 // this is for building LogicalCompatibilityContext later. - private final Map> shuttledExpressionsToExpressionsMap; + private final Map>> + shuttledExpressionsToExpressionsMap; + // For value of Map, the key is the position of expression + // the value is the original expression and shuttled expression map + // Such as origin expressions are l_partkey#0 > 1 and shuttled expression is l_partkey#10 > 1 + // the map would be {ExpressionPosition.FILTER, { + // l_partkey#0 > 1 : l_partkey#10 > 1 + // }} + // this is for building LogicalCompatibilityContext later. + private final Map> expressionToShuttledExpressionToMap; + // Record the exprId and the corresponding expr map, this is used by expression shuttled private final Map namedExprIdAndExprMapping; private final List planOutputShuttledExpressions; @@ -125,7 +142,9 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, - Map> shuttledExpressionsToExpressionsMap, + Map>> + shuttledExpressionsToExpressionsMap, + Map> expressionToShuttledExpressionToMap, Map namedExprIdAndExprMapping, BitSet tableIdSet, SplitPredicate splitPredicate, @@ -144,6 +163,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG this.splitPredicate = splitPredicate; this.equivalenceClass = equivalenceClass; this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap; + this.expressionToShuttledExpressionToMap = expressionToShuttledExpressionToMap; this.namedExprIdAndExprMapping = namedExprIdAndExprMapping; this.planOutputShuttledExpressions = planOutputShuttledExpressions; } @@ -154,7 +174,8 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, - this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet, + this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, + this.namedExprIdAndExprMapping, this.tableBitSet, null, null, this.planOutputShuttledExpressions); } @@ -164,13 +185,16 @@ public StructInfo withPredicates(Predicates predicates) { public StructInfo withTableBitSet(BitSet tableBitSet) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, this.predicates, - this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, tableBitSet, + this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, + this.namedExprIdAndExprMapping, tableBitSet, this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, Plan topPlan, - Map> shuttledExpressionsToExpressionsMap, + Map>> + shuttledExpressionsToExpressionsMap, + Map> expressionToShuttledExpressionToMap, Map namedExprIdAndExprMapping, List relations, Map relationIdStructInfoNodeMap, @@ -198,8 +222,9 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List replacedExpressions = replaceContext.getReplacedExpressions(); - putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, - ExpressionPosition.NODE, replacedExpressions.get(0), expression); + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, + ExpressionPosition.NODE, replacedExpressions.get(0), expression, node); // Record this, will be used in top level expression shuttle later, see the method // ExpressionLineageReplacer#visitGroupPlan namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); @@ -212,28 +237,27 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, }); // Collect expression from join condition in hyper graph for (JoinEdge edge : hyperGraph.getJoinEdges()) { - List hashJoinConjuncts = edge.getHashJoinConjuncts(); + List joinConjunctExpressions = edge.getExpressions(); // shuttle expression in edge for the build of LogicalCompatibilityContext later. // Record the exprId to expr map in the processing to strut info // TODO get exprId to expr map when complex project is ready in join dege - hashJoinConjuncts.forEach(conjunctExpr -> { - ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = - new ExpressionLineageReplacer.ExpressionReplaceContext( - Lists.newArrayList(conjunctExpr), ImmutableSet.of(), - ImmutableSet.of(), new BitSet()); - topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); - // Replace expressions by expression map - List replacedExpressions = replaceContext.getReplacedExpressions(); - putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, - ExpressionPosition.JOIN_EDGE, replacedExpressions.get(0), conjunctExpr); - // Record this, will be used in top level expression shuttle later, see the method - // ExpressionLineageReplacer#visitGroupPlan - namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); - }); - List otherJoinConjuncts = edge.getOtherJoinConjuncts(); - if (!otherJoinConjuncts.isEmpty()) { - return false; + ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = + new ExpressionLineageReplacer.ExpressionReplaceContext( + joinConjunctExpressions.stream().map(expr -> (Expression) expr) + .collect(Collectors.toList()), + ImmutableSet.of(), ImmutableSet.of(), new BitSet()); + topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); + // Replace expressions by expression map + List replacedExpressions = replaceContext.getReplacedExpressions(); + for (int i = 0; i < replacedExpressions.size(); i++) { + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, + ExpressionPosition.JOIN_EDGE, replacedExpressions.get(i), joinConjunctExpressions.get(i), + edge); } + // Record this, will be used in top level expression shuttle later, see the method + // ExpressionLineageReplacer#visitGroupPlan + namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); } // Collect expression from where in hyper graph hyperGraph.getFilterEdges().forEach(filterEdge -> { @@ -241,10 +265,11 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, filterExpressions.forEach(predicate -> { // this is used for LogicalCompatibilityContext ExpressionUtils.extractConjunction(predicate).forEach(expr -> - putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, ExpressionPosition.FILTER_EDGE, ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, new BitSet()), - predicate)); + predicate, filterEdge)); }); }); return true; @@ -316,11 +341,13 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable // collect struct info fromGraph List relationList = new ArrayList<>(); Map relationIdStructInfoNodeMap = new LinkedHashMap<>(); - Map> shuttledHashConjunctsToConjunctsMap = - new LinkedHashMap<>(); + Map>> + shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); Map namedExprIdAndExprMapping = new LinkedHashMap<>(); BitSet tableBitSet = new BitSet(); + Map> expressionToShuttledExpressionToMap = new HashMap<>(); boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, + expressionToShuttledExpressionToMap, namedExprIdAndExprMapping, relationList, relationIdStructInfoNodeMap, @@ -342,19 +369,11 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(), originalPlan, new BitSet()); return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, predicates, shuttledHashConjunctsToConjunctsMap, + expressionToShuttledExpressionToMap, namedExprIdAndExprMapping, tableBitSet, null, null, planOutputShuttledExpressions); } - /** - * Build Struct info from group. - * Maybe return multi structInfo when original plan already be rewritten by mv - */ - public static StructInfo of(Group group) { - // TODO build graph from original plan and get relations and predicates from graph - return null; - } - public List getRelations() { return relations; } @@ -411,21 +430,36 @@ public Map getRelationIdStructInfoNodeMap() { return relationIdStructInfoNodeMap; } - public Map> getShuttledExpressionsToExpressionsMap() { + public Map>> + getShuttledExpressionsToExpressionsMap() { return shuttledExpressionsToExpressionsMap; } - private static void putShuttledExpressionsToExpressionsMap( - Map> shuttledExpressionsToExpressionsMap, + public Map> getExpressionToShuttledExpressionToMap() { + return expressionToShuttledExpressionToMap; + } + + private static void putShuttledExpressionToExpressionsMap( + Map>> + shuttledExpressionsToExpressionsMap, + Map> expressionPositionToExpressionToMap, ExpressionPosition expressionPosition, - Expression key, Expression value) { - Map expressionExpressionMap = shuttledExpressionsToExpressionsMap.get( - expressionPosition); - if (expressionExpressionMap == null) { - expressionExpressionMap = new LinkedHashMap<>(); - shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap); - } - expressionExpressionMap.put(key, value); + Expression shuttledExpression, Expression originalExpression, HyperElement valueBelongedElement) { + Multimap> shuttledExpressionToExpressionMap = + shuttledExpressionsToExpressionsMap.get(expressionPosition); + if (shuttledExpressionToExpressionMap == null) { + shuttledExpressionToExpressionMap = HashMultimap.create(); + shuttledExpressionsToExpressionsMap.put(expressionPosition, shuttledExpressionToExpressionMap); + } + shuttledExpressionToExpressionMap.put(shuttledExpression, Pair.of(originalExpression, valueBelongedElement)); + + Map originalExprToShuttledExprMap = + expressionPositionToExpressionToMap.get(expressionPosition); + if (originalExprToShuttledExprMap == null) { + originalExprToShuttledExprMap = new HashMap<>(); + expressionPositionToExpressionToMap.put(expressionPosition, originalExprToShuttledExprMap); + } + originalExprToShuttledExprMap.put(originalExpression, shuttledExpression); } public List getExpressions() { diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out index 5c9df6b7f92256..c400e078daff1a 100644 --- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out +++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out @@ -323,3 +323,27 @@ c 3 6 c,c,c 5.333333333333333 mi 3 2 1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 +-- !query29_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query29_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query30_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query30_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query31_0_before -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + +-- !query31_0_after -- +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 \N \N 8 8 1 +1 2023-12-09 1 yy 2 2 2 4 3 \N 2 3 1 2 8 8 1 + diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index 83f924a7ea5c65..bed1f1dcaa39ce 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -45,11 +45,9 @@ suite("aggregate_without_roll_up") { "replication_num" = "1" ); """ - sql """ drop table if exists lineitem """ - sql""" CREATE TABLE IF NOT EXISTS lineitem ( l_orderkey INTEGER NOT NULL, @@ -76,11 +74,9 @@ suite("aggregate_without_roll_up") { "replication_num" = "1" ) """ - sql """ drop table if exists partsupp """ - sql """ CREATE TABLE IF NOT EXISTS partsupp ( ps_partkey INTEGER NOT NULL, @@ -1513,4 +1509,463 @@ suite("aggregate_without_roll_up") { order_qt_query28_0_after "${query28_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv28_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully + def mv29_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query29_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query29_0_before "${query29_0}" + async_mv_rewrite_success(db, mv29_0, query29_0, "mv29_0") + order_qt_query29_0_after "${query29_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully + // mv join condition has alias + def mv30_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey_alias, + orders.public_col as col1, + l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey_alias, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2, + sum( + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey as o_orderkey_alias, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey as l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias + inner join ( + select + ps_partkey as ps_partkey_alias, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey_alias = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query30_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query30_0_before "${query30_0}" + async_mv_rewrite_success(db, mv30_0, query30_0, "mv30_0") + order_qt_query30_0_after "${query30_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv30_0""" + + + // query and mv has the same filter but position is different, should rewrite successfully + // query join condition has alias + def mv31_0 = """ + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as col1, + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey + l_orderkey + ps_partkey * 2, + sum( + o_orderkey + l_orderkey + ps_partkey * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on l_orderkey = o_orderkey + inner join ( + select + ps_partkey, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on ps_partkey = o_orderkey + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + def query31_0 = """ +select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey_alias, + orders.public_col as col1, + l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as col2, + ps_partkey_alias, + ps_suppkey, + partsupp.public_col as col3, + partsupp.public_col * 2 as col4, + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2, + sum( + o_orderkey_alias + l_orderkey_alias + ps_partkey_alias * 2 + ), + count() as count_all + from + ( + select + o_custkey, + o_orderdate, + o_shippriority, + o_comment, + o_orderkey as o_orderkey_alias, + orders.public_col as public_col + from + orders + ) orders + left join ( + select + l_orderkey as l_orderkey_alias, + l_partkey, + l_suppkey, + lineitem.public_col as public_col + from + lineitem + where + lineitem.public_col is null + or lineitem.public_col <> 1 + ) lineitem on lineitem.l_orderkey_alias = orders.o_orderkey_alias + inner join ( + select + ps_partkey as ps_partkey_alias, + ps_suppkey, + partsupp.public_col as public_col + from + partsupp + ) partsupp on partsupp.ps_partkey_alias = orders.o_orderkey_alias + where + lineitem.public_col is null + or lineitem.public_col <> 1 + and o_orderkey_alias = 2 + group by + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14; + """ + order_qt_query31_0_before "${query31_0}" + async_mv_rewrite_success(db, mv31_0, query31_0, "mv31_0") + order_qt_query31_0_after "${query31_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv31_0""" } From e94e54c14ad28556b471e6f34f9beda7c4270722 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 5 Dec 2024 12:08:50 +0800 Subject: [PATCH 5/7] [opt](mtmv) Optimize plan generate when create mtmv and use mtmv cache when collect table of mtmv (#44786) Optimize plan generate when create mtmv and use mtmv cache when collect table of mtmv 1. Reuse plans when creating materialized views to minimize plan generation overhead. 2. During recursive base table resolution for MTMVs, prioritize MTMV cache lookup. Fall back to real-time generation only when cache miss occurs. --- .../java/org/apache/doris/mtmv/MTMVCache.java | 14 +++- .../org/apache/doris/mtmv/MTMVPlanUtil.java | 31 +++----- .../mv/InitMaterializationContextHook.java | 8 +-- .../exploration/mv/MaterializedViewUtils.java | 2 +- .../plans/commands/info/CreateMTMVInfo.java | 40 +++++------ .../info/MTMVPartitionDefinition.java | 71 +++++++------------ .../trees/plans/visitor/TableCollector.java | 34 +++++---- .../nereids/trees/plans/PlanVisitorTest.java | 14 ++-- 8 files changed, 94 insertions(+), 120 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 660ebde293acbb..d3d7f1ad6ebbf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -51,14 +51,18 @@ public class MTMVCache { // The materialized view plan which should be optimized by the same rules to query // and will remove top sink and unused sort private final Plan logicalPlan; - // The original plan of mv def sql + // The original rewritten plan of mv def sql private final Plan originalPlan; + // The analyzed plan of mv def sql, which is used by tableCollector,should not be optimized by rbo + private final Plan analyzedPlan; private final Statistics statistics; private final StructInfo structInfo; - public MTMVCache(Plan logicalPlan, Plan originalPlan, Statistics statistics, StructInfo structInfo) { + public MTMVCache(Plan logicalPlan, Plan originalPlan, Plan analyzedPlan, + Statistics statistics, StructInfo structInfo) { this.logicalPlan = logicalPlan; this.originalPlan = originalPlan; + this.analyzedPlan = analyzedPlan; this.statistics = statistics; this.structInfo = structInfo; } @@ -71,6 +75,10 @@ public Plan getOriginalPlan() { return originalPlan; } + public Plan getAnalyzedPlan() { + return analyzedPlan; + } + public Statistics getStatistics() { return statistics; } @@ -118,7 +126,7 @@ public Plan visitLogicalResultSink(LogicalResultSink logicalResu Optional structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan, planner.getCascadesContext(), new BitSet()); - return new MTMVCache(mvPlan, originPlan, needCost + return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), needCost ? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null, structInfoOptional.orElseGet(() -> null)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index e2c64b5e5b7c94..5761b41171e644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -35,12 +35,10 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.TableCollector; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -106,31 +104,20 @@ private static void setCatalogAndDb(ConnectContext ctx, MTMV mtmv) { public static MTMVRelation generateMTMVRelation(MTMV mtmv, ConnectContext ctx) { // Should not make table without data to empty relation when analyze the related table, // so add disable rules - SessionVariable sessionVariable = ctx.getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - Plan plan; - try { - plan = getPlanBySql(mtmv.getQuerySql(), ctx); - } finally { - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - return generateMTMVRelation(plan); + Plan plan = getAnalyzePlanBySql(mtmv.getQuerySql(), ctx); + return generateMTMVRelation(plan, ctx); } - public static MTMVRelation generateMTMVRelation(Plan plan) { - return new MTMVRelation(getBaseTables(plan, true), getBaseTables(plan, false), getBaseViews(plan)); + public static MTMVRelation generateMTMVRelation(Plan plan, ConnectContext connectContext) { + return new MTMVRelation(getBaseTables(plan, true, connectContext), + getBaseTables(plan, false, connectContext), getBaseViews(plan)); } - private static Set getBaseTables(Plan plan, boolean expand) { + private static Set getBaseTables(Plan plan, boolean expand, ConnectContext connectContext) { TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( com.google.common.collect.Sets - .newHashSet(TableType.values()), expand); + .newHashSet(TableType.values()), expand, connectContext); plan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); return transferTableIfToInfo(collectedTables); @@ -148,7 +135,7 @@ private static Set transferTableIfToInfo(Set tables) { return result; } - private static Plan getPlanBySql(String querySql, ConnectContext ctx) { + private static Plan getAnalyzePlanBySql(String querySql, ConnectContext ctx) { List statements; try { statements = new NereidsParser().parseSQL(querySql); @@ -161,7 +148,7 @@ private static Plan getPlanBySql(String querySql, ConnectContext ctx) { ctx.setStatementContext(new StatementContext()); try { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.NONE); + return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); } finally { ctx.setStatementContext(original); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 37d66b101cfa4f..925e8dba3c31fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -75,12 +75,12 @@ protected void doInitMaterializationContext(CascadesContext cascadesContext) { return; } // Only collect the table or mv which query use directly, to avoid useless mv partition in rewrite - TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false); + // Keep use one connection context when in query, if new connect context, + // the ConnectionContext.get() will change + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), false, + cascadesContext.getConnectContext()); try { Plan rewritePlan = cascadesContext.getRewritePlan(); - // Keep use one connection context when in query, if new connect context, - // the ConnectionContext.get() will change - collectorContext.setConnectContext(cascadesContext.getConnectContext()); rewritePlan.accept(TableCollector.INSTANCE, collectorContext); } catch (Exception e) { LOG.warn(String.format("MaterializationContext init table collect fail, current queryId is %s", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 8495349c643f54..f87f67d2e7a77c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -343,7 +343,7 @@ public Plan visitLogicalResultSink(LogicalResultSink logicalResu ImmutableList.of(Rewriter.custom(RuleType.ELIMINATE_SORT, EliminateSort::new))).execute(); return childContext.getRewritePlan(); }, mvPlan, originPlan); - return new MTMVCache(mvPlan, originPlan, + return new MTMVCache(mvPlan, originPlan, planner.getAnalyzedPlan(), planner.getCascadesContext().getMemo().getRoot().getStatistics(), null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index ed3269d7142bb5..adfb06e7d80010 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -255,9 +255,21 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t NereidsPlanner planner = new NereidsPlanner(statementContext); // this is for expression column name infer when not use alias LogicalSink logicalSink = new UnboundResultSink<>(logicalQuery); - // must disable constant folding by be, because be constant folding may return wrong type - ctx.getSessionVariable().disableConstantFoldingByBEOnce(); - Plan plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + // Should not make table without data to empty relation when analyze the related table, + // so add disable rules + Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); + ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + Plan plan; + try { + // must disable constant folding by be, because be constant folding may return wrong type + ctx.getSessionVariable().disableConstantFoldingByBEOnce(); + plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + } finally { + // after operate, roll back the disable rules + ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); + ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } // can not contain VIEW or MTMV analyzeBaseTables(planner.getAnalyzedPlan()); // can not contain Random function @@ -268,8 +280,7 @@ public void analyzeQuery(ConnectContext ctx, Map mvProperties) t throw new AnalysisException("can not contain invalid expression"); } getRelation(planner); - this.mvPartitionInfo = mvPartitionDefinition - .analyzeAndTransferToMTMVPartitionInfo(planner, ctx, logicalQuery); + this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner, ctx); this.partitionDesc = generatePartitionDesc(ctx); getColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), distribution); analyzeKeys(); @@ -314,24 +325,9 @@ private void analyzeKeys() { } } + // Should use analyzed plan for collect views and tables private void getRelation(NereidsPlanner planner) { - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - ConnectContext ctx = planner.getCascadesContext().getConnectContext(); - SessionVariable sessionVariable = ctx.getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - if (ctx.getStatementContext() != null) { - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - Plan plan; - try { - plan = planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.NONE); - } finally { - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - this.relation = MTMVPlanUtil.generateMTMVRelation(plan); + this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan(), planner.getConnectContext()); } private PartitionDesc generatePartitionDesc(ConnectContext ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java index c4117e8608e29d..a26a97f7240793 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; import org.apache.doris.nereids.trees.expressions.Cast; @@ -45,11 +44,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Sets; @@ -72,11 +67,9 @@ public class MTMVPartitionDefinition { * * @param planner planner * @param ctx ctx - * @param logicalQuery logicalQuery * @return MTMVPartitionInfo */ - public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, - LogicalPlan logicalQuery) { + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx) { MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { return mtmvPartitionInfo; @@ -100,7 +93,7 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl timeUnit = null; } mtmvPartitionInfo.setPartitionCol(partitionColName); - RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName, timeUnit); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, partitionColName, timeUnit); mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); if (relatedTableInfo.getPartitionExpression().isPresent()) { @@ -125,47 +118,33 @@ public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner pl return mtmvPartitionInfo; } - private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan - logicalQuery, - String partitionColName, - String timeUnit) { + // Should use rewritten plan without view and subQuery to get related partition table + private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, + String partitionColName, String timeUnit) { CascadesContext cascadesContext = planner.getCascadesContext(); - SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + + RelatedTableInfo relatedTableInfo = MaterializedViewUtils + .getRelatedTableInfo(partitionColName, timeUnit, planner.getRewrittenPlan(), cascadesContext); + if (!relatedTableInfo.isPctPossible()) { + throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," + + " the fail reason is %s", relatedTableInfo.getFailReason())); + } + MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); + Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); try { - Plan mvRewrittenPlan = - planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); - RelatedTableInfo relatedTableInfo = MaterializedViewUtils - .getRelatedTableInfo(partitionColName, timeUnit, mvRewrittenPlan, cascadesContext); - if (!relatedTableInfo.isPctPossible()) { - throw new AnalysisException(String.format("Unable to find a suitable base table for partitioning," - + " the fail reason is %s", relatedTableInfo.getFailReason())); - } - MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo()); - Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - try { - partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty())); - } catch (DdlException e) { - throw new AnalysisException(e.getMessage(), e); - } + partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty())); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage(), e); + } - if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { - throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); - } - if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) - && partitionColumnNames.size() != 1) { - throw new AnalysisException("only hms table support multi column partition."); - } - return relatedTableInfo; - } finally { - // after operate, roll back the disable rules - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + if (!partitionColumnNames.contains(relatedTableInfo.getColumn())) { + throw new AnalysisException("error related column: " + relatedTableInfo.getColumn()); + } + if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) + && partitionColumnNames.size() != 1) { + throw new AnalysisException("only hms table support multi column partition."); } + return relatedTableInfo; } private static List convertToLegacyArguments(List children) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 2e2cdb810f0f72..27ff1e4b68c075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -20,8 +20,8 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVCache; -import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; @@ -70,13 +70,19 @@ public Plan visitPhysicalCatalogRelation(PhysicalCatalogRelation catalogRelation } private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) { - if (!context.isExpand()) { + if (!context.isExpandMaterializedView()) { return; } // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong - MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null - ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext(), false); - expandedMv.getLogicalPlan().accept(this, context); + MTMVCache expandedMvCache; + try { + expandedMvCache = mtmv.getOrGenerateCache(context.getConnectContext()); + } catch (AnalysisException exception) { + LOG.warn(String.format("expandMvAndCollect getOrGenerateCache fail, mtmv name is %s", mtmv.getName()), + exception); + expandedMvCache = MTMVCache.from(mtmv, context.getConnectContext(), false); + } + expandedMvCache.getAnalyzedPlan().accept(this, context); } /** @@ -87,12 +93,14 @@ public static final class TableCollectorContext { private final Set collectedTables = new HashSet<>(); private final Set targetTableTypes; // if expand the mv or not - private final boolean expand; - private ConnectContext connectContext; + private final boolean expandMaterializedView; + private final ConnectContext connectContext; - public TableCollectorContext(Set targetTableTypes, boolean expand) { + public TableCollectorContext(Set targetTableTypes, boolean expandMaterializedView, + ConnectContext connectContext) { this.targetTableTypes = targetTableTypes; - this.expand = expand; + this.expandMaterializedView = expandMaterializedView; + this.connectContext = connectContext; } public Set getCollectedTables() { @@ -103,16 +111,12 @@ public Set getTargetTableTypes() { return targetTableTypes; } - public boolean isExpand() { - return expand; + public boolean isExpandMaterializedView() { + return expandMaterializedView; } public ConnectContext getConnectContext() { return connectContext; } - - public void setConnectContext(ConnectContext connectContext) { - this.connectContext = connectContext; - } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index 5fbc6d7b7ac254..9c56bb1186a5ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -130,7 +130,7 @@ public void test1() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -159,7 +159,7 @@ public void test2() { Assertions.assertTrue(nondeterministicFunctionSet.get(1) instanceof Random); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -196,7 +196,7 @@ public BitSet getDisableNereidsRules() { Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Uuid); // Check get tables TableCollectorContext collectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.OLAP), true); + Sets.newHashSet(TableType.OLAP), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContext); Set expectedTables = new HashSet<>(); expectedTables.add("table1"); @@ -210,7 +210,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext collectorContextWithNoExpand = new TableCollector.TableCollectorContext(Sets.newHashSet(TableType.OLAP), - false); + false, connectContext); physicalPlan.accept(TableCollector.INSTANCE, collectorContextWithNoExpand); Set expectedTablesWithNoExpand = new HashSet<>(); expectedTablesWithNoExpand.add("table1"); @@ -222,7 +222,7 @@ public BitSet getDisableNereidsRules() { expectedTablesWithNoExpand); TableCollectorContext mvCollectorContext = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), true); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContext); Set expectedMvs = new HashSet<>(); expectedMvs.add("mv1"); @@ -234,7 +234,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext mvCollectorContextWithNoExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.MATERIALIZED_VIEW), false); + Sets.newHashSet(TableType.MATERIALIZED_VIEW), false, connectContext); physicalPlan.accept(TableCollector.INSTANCE, mvCollectorContextWithNoExpand); Set expectedMvsWithNoExpand = new HashSet<>(); expectedMvsWithNoExpand.add("mv1"); @@ -246,7 +246,7 @@ public BitSet getDisableNereidsRules() { TableCollectorContext allTableTypeWithExpand = new TableCollector.TableCollectorContext( - Sets.newHashSet(TableType.values()), true); + Sets.newHashSet(TableType.values()), true, connectContext); physicalPlan.accept(TableCollector.INSTANCE, allTableTypeWithExpand); // when collect in plan with expand, should collect table which is expended Set expectedTablesWithExpand = new HashSet<>(); From bc8258a936835eb968bfe032d581ec8f8af74021 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 5 Dec 2024 12:07:50 +0800 Subject: [PATCH 6/7] [fix](mtmv) Fix refresh materialized view fail when mv def contains cte (#44857) ### What problem does this PR solve? Related PR: #33988 Problem Summary: if mv def contain cte and is partition mv, such as following: CREATE MATERIALIZED VIEW mv_name BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL PARTITION BY (l_shipdate) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS WITH scan_data_cte as ( select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey from (select * from lineitem where L_LINENUMBER > 1) t1 left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY ) SELECT * FROM scan_data_cte; if run refresh cmd as following, this would fail, throw exception `no partition for this tuple`, this pr fix this refresh materialized view mv_name partitions(p_20231210_20231211); ### Release note Fix refresh materialized view fail when mv def contains cte --- .../commands/UpdateMvByPartitionCommand.java | 9 +- .../mv/micro_test/micro_test_when_cte.out | 128 +++++++++++ .../mv/micro_test/micro_test_when_cte.groovy | 204 ++++++++++++++++++ 3 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out create mode 100644 regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index de284bd837748f..36cc0f95a77a8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -246,11 +246,16 @@ public Plan visitLogicalCTE(LogicalCTE cte, PredicateAddContext if (predicates.isEmpty()) { return cte; } + List> rewrittenSubQueryAlias = new ArrayList<>(); for (LogicalSubQueryAlias subQueryAlias : cte.getAliasQueries()) { + List subQueryAliasChildren = new ArrayList<>(); this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier()); - subQueryAlias.children().forEach(subQuery -> subQuery.accept(this, predicates)); + subQueryAlias.children().forEach(subQuery -> + subQueryAliasChildren.add(subQuery.accept(this, predicates)) + ); + rewrittenSubQueryAlias.add(subQueryAlias.withChildren(subQueryAliasChildren)); } - return super.visitLogicalCTE(cte, predicates); + return super.visitLogicalCTE(new LogicalCTE<>(rewrittenSubQueryAlias, cte.child()), predicates); } @Override diff --git a/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out new file mode 100644 index 00000000000000..8abaaf9adec0a8 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/micro_test/micro_test_when_cte.out @@ -0,0 +1,128 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_0_after -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_0 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_1 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_0_after -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_2 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + +-- !query_mv_3 -- +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-08 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-09 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 3 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-10 4 1 4 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-11 4 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 +2023-12-12 6 2 3 + diff --git a/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy new file mode 100644 index 00000000000000..9e80ea966fc1a2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/micro_test/micro_test_when_cte.groovy @@ -0,0 +1,204 @@ +package mv.micro_test +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("micro_test_when_cte") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate)( + FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) + (FROM ('2023-12-01') TO ('2023-12-31') INTERVAL 1 DAY) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + sql """analyze table partsupp with sync""" + sql """analyze table lineitem with sync""" + sql """analyze table orders with sync""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='5');""" + sql """alter table orders modify column O_COMMENT set stats ('row_count'='18');""" + sql """alter table partsupp modify column ps_comment set stats ('row_count'='2');""" + + def query_sql = """ + WITH scan_data_cte as ( + select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey + from (select * from lineitem where L_LINENUMBER > 1) t1 + left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY + ) + SELECT * FROM scan_data_cte; + """ + def mv_sql = """ + WITH scan_data_cte as ( + select t1.l_shipdate, t1.L_LINENUMBER, orders.O_CUSTKEY, l_suppkey + from (select * from lineitem where L_LINENUMBER > 1) t1 + left join orders on t1.L_ORDERKEY = orders.O_ORDERKEY + ) + SELECT * FROM scan_data_cte; + """ + def mv_name = """mv_with_cte_test""" + + // query directly + order_qt_query_0_after "${query_sql}" + + // create and build complete mv + create_async_mv(db, mv_name, mv_sql) + // refresh mv complete + sql """refresh materialized view ${mv_name} complete""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_0 "select * from ${mv_name}" + + // create and build partition mv + create_async_partition_mv(db, mv_name, mv_sql, "(l_shipdate)") + + // refresh mv partly + sql """refresh materialized view ${mv_name} partitions(p_20231208_20231209)""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_1 "select * from ${mv_name}" + + // query rewrite + mv_rewrite_success(mv_sql, mv_name) + order_qt_query_0_after "${query_sql}" + + // DML + // base table insert into data when not partition table + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'); + """ + sql """refresh materialized view ${mv_name} complete""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_2 "select * from ${mv_name}" + + // base table insert into data when partition table + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'); + """ + sql """refresh materialized view ${mv_name} partitions(p_20231210_20231211)""" + // query mv directly + waitingMTMVTaskFinishedByMvName(mv_name) + order_qt_query_mv_3 "select * from ${mv_name}" +} From f712bd4566044e38d33a334546a6ff405136adc3 Mon Sep 17 00:00:00 2001 From: seawinde Date: Fri, 6 Dec 2024 20:48:32 +0800 Subject: [PATCH 7/7] fix code usage --- .../nereids/trees/plans/commands/info/CreateMTMVInfo.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index adfb06e7d80010..867b22dcde746e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -327,7 +327,8 @@ private void analyzeKeys() { // Should use analyzed plan for collect views and tables private void getRelation(NereidsPlanner planner) { - this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan(), planner.getConnectContext()); + this.relation = MTMVPlanUtil.generateMTMVRelation(planner.getAnalyzedPlan(), + planner.getCascadesContext().getConnectContext()); } private PartitionDesc generatePartitionDesc(ConnectContext ctx) {