From c34de355e1d16a62a8b12708c6bb27ad940b799f Mon Sep 17 00:00:00 2001 From: liutang123 Date: Thu, 25 Jul 2024 16:27:21 +0800 Subject: [PATCH 1/4] [fix] (mtmv) Choose a valid partition column when there are both valid and invalid expressions --- .../exploration/mv/MaterializedViewUtils.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index f5cfb590d9be3e..ac336cc22af756 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -589,7 +589,8 @@ private SlotReference getContextPartitionColumn(IncrementCheckerContext context) private static boolean checkPartition(Collection expressionsToCheck, Plan plan, IncrementCheckerContext context) { NamedExpression partitionColumn = context.getMvPartitionColumn(); - for (Expression projectSlot : expressionsToCheck) { + + OUTER_CHECK: for (Expression projectSlot : expressionsToCheck) { if (projectSlot.isColumnFromTable() && projectSlot.equals(partitionColumn.toSlot())) { continue; } @@ -621,7 +622,7 @@ private static boolean checkPartition(Collection expressio String.format("partition expression use more than one slot reference, invalid " + "expressionToCheckColumns is %s, partitionColumnDateColumns is %s", expressionToCheckColumns, partitionColumns)); - return false; + continue; } List expressions = expressionToCheck.collectToList(Expression.class::isInstance); for (Expression expression : expressions) { @@ -630,7 +631,7 @@ private static boolean checkPartition(Collection expressio context.addFailReason( String.format("column to check use invalid implicit expression, invalid " + "expression is %s", expression)); - return false; + continue OUTER_CHECK; } } List partitionExpressions = partitionExpression.collectToList( @@ -641,7 +642,7 @@ private static boolean checkPartition(Collection expressio context.addFailReason( String.format("partition column use invalid implicit expression, invalid " + "expression is %s", expression)); - return false; + continue OUTER_CHECK; } } List expressionToCheckDataTruncList = @@ -652,7 +653,7 @@ private static boolean checkPartition(Collection expressio // mv time unit level is little then query context.addFailReason("partition column time unit level should be " + "greater than sql select column"); - return false; + continue; } if (!partitionColumn.isColumnFromTable()) { context.setMvPartitionColumn(partitionColumns.iterator().next()); @@ -660,8 +661,9 @@ private static boolean checkPartition(Collection expressio if (!context.getPartitionExpression().isPresent()) { context.setPartitionExpression(partitionExpression); } + return true; } - return true; + return false; } } From efe296e44132175d269e18f5b6d00c85b35897f8 Mon Sep 17 00:00:00 2001 From: liutang123 Date: Mon, 29 Jul 2024 11:12:46 +0800 Subject: [PATCH 2/4] add ut --- .../mv/MaterializedViewUtilsTest.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 89fe34876ae476..9ac6d7d3a753ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -234,6 +234,19 @@ protected void runBeforeAll() throws Exception { + " \"replication_num\" = \"1\"\n" + ");\n" ); + createTable("CREATE TABLE `test3` (\n" + + " `id` VARCHAR(36) NOT NULL COMMENT 'id',\n" + + " `created_time` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT ''\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`id`)\n" + + "COMMENT ''\n" + + "PARTITION BY RANGE(`created_time`)\n" + + "(PARTITION P_2024071713 VALUES [('2024-07-17 13:00:00'), ('2024-07-17 14:00:00')),\n" + + "PARTITION P_2024071714 VALUES [('2024-07-17 14:00:00'), ('2024-07-17 15:00:00')))\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS AUTO\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\"\n" + + ");\n"); // Should not make scan to empty relation when the table used by materialized view has no data connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION"); } @@ -809,6 +822,42 @@ public void containTableQueryOperatorWithoutOperatorTest() { }); } + @Test + public void getRelatedTableInfoWhenMultiPartitionExprs() { + PlanChecker.from(connectContext) + .checkExplain("select id, date_trunc(created_time, 'minute') as created_time_minute," + + " min(created_time) as start_time," + + " if(count(id) > 0, 1, 0) as status\n" + + " from test3 \n" + + " group by id, date_trunc(created_time, 'minute')", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("created_time_minute", + "day", rewrittenPlan, nereidsPlanner.getCascadesContext()); + checkRelatedTableInfo(relatedTableInfo, + "test3", + "created_time", + true); + }); + PlanChecker.from(connectContext) + .checkExplain("select id, date_trunc(created_time, 'hour') as created_time_hour," + + " min(created_time) as start_time\n" + + " from test3 \n" + + " group by id, date_trunc(created_time, 'minute')," + + " date_trunc(created_time, 'hour');", + nereidsPlanner -> { + Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan(); + RelatedTableInfo relatedTableInfo = + MaterializedViewUtils.getRelatedTableInfo("created_time_hour", + "day", rewrittenPlan, nereidsPlanner.getCascadesContext()); + checkRelatedTableInfo(relatedTableInfo, + "test3", + "created_time", + true); + }); + } + private void checkRelatedTableInfo(RelatedTableInfo relatedTableInfo, String expectTableName, String expectColumnName, From ce8cccedc19365b0209c98b9f06298a17a781897 Mon Sep 17 00:00:00 2001 From: liutang123 Date: Mon, 29 Jul 2024 23:35:05 +0800 Subject: [PATCH 3/4] fix ut --- .../rules/exploration/mv/MaterializedViewUtils.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index ac336cc22af756..9efa4fd1b07c2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -592,7 +592,12 @@ private static boolean checkPartition(Collection expressio OUTER_CHECK: for (Expression projectSlot : expressionsToCheck) { if (projectSlot.isColumnFromTable() && projectSlot.equals(partitionColumn.toSlot())) { - continue; + if (!context.getPartitionExpression().isPresent()) { + context.setPartitionExpression( + ExpressionUtils.shuttleExpressionWithLineage(partitionColumn, plan, new BitSet()) + ); + } + return true; } // check the expression which use partition column Expression expressionToCheck = From 3b2260f38afec6bfe59d51dc27f1a6e5695eea66 Mon Sep 17 00:00:00 2001 From: liutang123 Date: Tue, 30 Jul 2024 17:10:02 +0800 Subject: [PATCH 4/4] small fix --- .../rules/exploration/mv/MaterializedViewUtils.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 9efa4fd1b07c2e..29f778c35d1990 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -592,12 +592,7 @@ private static boolean checkPartition(Collection expressio OUTER_CHECK: for (Expression projectSlot : expressionsToCheck) { if (projectSlot.isColumnFromTable() && projectSlot.equals(partitionColumn.toSlot())) { - if (!context.getPartitionExpression().isPresent()) { - context.setPartitionExpression( - ExpressionUtils.shuttleExpressionWithLineage(partitionColumn, plan, new BitSet()) - ); - } - return true; + continue; } // check the expression which use partition column Expression expressionToCheck = @@ -668,7 +663,7 @@ private static boolean checkPartition(Collection expressio } return true; } - return false; + return context.getMvPartitionColumn().isColumnFromTable(); } }