From 5ccbd2db236c5e298f1b844c75034dee8160125a Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 11 Dec 2024 12:13:12 +0800 Subject: [PATCH 1/5] 1 --- .../plans/commands/info/RefreshMTMVInfo.java | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java index bf483f87a152c3..67466a8a6ccd7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.commands.info; +import org.apache.doris.analysis.AllPartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; @@ -24,16 +26,20 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVPartitionUtil; +import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import org.apache.commons.collections.CollectionUtils; +import org.glassfish.jersey.internal.guava.Sets; import java.util.List; import java.util.Objects; +import java.util.Set; /** * refresh mtmv info @@ -67,13 +73,40 @@ public void analyze(ConnectContext ctx) { Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb()); MTMV mtmv = (MTMV) db.getTableOrMetaException(mvName.getTbl(), TableType.MATERIALIZED_VIEW); if (!CollectionUtils.isEmpty(partitions)) { - MTMVPartitionUtil.getPartitionsIdsByNames(mtmv, partitions); + checkPartitionExist(mtmv); } } catch (org.apache.doris.common.AnalysisException | MetaNotFoundException | DdlException e) { throw new AnalysisException(e.getMessage()); } } + private void checkPartitionExist(MTMV mtmv) throws org.apache.doris.common.AnalysisException { + MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); + // todo: Unified locking sequence + mtmv.readLock(); + relatedTable.readLock(); + try { + if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { + throw new AnalysisException( + "The partition method of this asynchronous materialized view does not support refreshing by partition"); + } + List partitionDescs = MTMVPartitionUtil.getPartitionDescsByRelatedTable( + mtmv.getTableProperty().getProperties(), mtmv.getMvPartitionInfo(), mtmv.getMvProperties()); + Set shouldExistPartitionNames = Sets.newHashSetWithExpectedSize(partitionDescs.size()); + partitionDescs.stream().forEach(desc -> { + shouldExistPartitionNames.add(((SinglePartitionDesc) desc).getPartitionName()); + }); + for (String partition : partitions) { + if (!shouldExistPartitionNames.contains(partition)) { + throw new org.apache.doris.common.AnalysisException("partition not exist: " + partition); + } + } + } finally { + relatedTable.readUnlock(); + mtmv.readUnlock(); + } + } + /** * getMvName * From 95fa966746d41211621f2e3ded87e42b88178551 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 11 Dec 2024 14:47:51 +0800 Subject: [PATCH 2/5] 1 --- .../nereids/trees/plans/commands/info/RefreshMTMVInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java index 67466a8a6ccd7e..738c3c1f3a0001 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java @@ -34,8 +34,8 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; -import org.glassfish.jersey.internal.guava.Sets; import java.util.List; import java.util.Objects; From c6978070d44a71646a0c17ab68359cd8e92829a3 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 11 Dec 2024 15:43:18 +0800 Subject: [PATCH 3/5] 1 --- .../nereids/trees/plans/commands/info/RefreshMTMVInfo.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java index 738c3c1f3a0001..422567cf865832 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java @@ -88,7 +88,8 @@ private void checkPartitionExist(MTMV mtmv) throws org.apache.doris.common.Analy try { if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { throw new AnalysisException( - "The partition method of this asynchronous materialized view does not support refreshing by partition"); + "The partition method of this asynchronous materialized view " + + "does not support refreshing by partition"); } List partitionDescs = MTMVPartitionUtil.getPartitionDescsByRelatedTable( mtmv.getTableProperty().getProperties(), mtmv.getMvPartitionInfo(), mtmv.getMvProperties()); From 605cbbc2653cff9b5e442c4e4458f8b5c989cec3 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 11 Dec 2024 16:00:21 +0800 Subject: [PATCH 4/5] add case --- .../test_refresh_partition_name_mtmv.groovy | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 regression-test/suites/mtmv_p0/test_refresh_partition_name_mtmv.groovy diff --git a/regression-test/suites/mtmv_p0/test_refresh_partition_name_mtmv.groovy b/regression-test/suites/mtmv_p0/test_refresh_partition_name_mtmv.groovy new file mode 100644 index 00000000000000..a18945339838d9 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_refresh_partition_name_mtmv.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_refresh_partition_name_mtmv","mtmv") { + String suiteName = "test_refresh_partition_name_mtmv" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 TINYINT, + k3 INT not null + ) + COMMENT "my first table" + PARTITION BY LIST(`k3`) + ( + PARTITION `p1` VALUES IN ('1'), + PARTITION `p2` VALUES IN ('2'), + PARTITION `p3` VALUES IN ('3') + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`k3`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'refresh_partition_num' = '2' + ) + AS + SELECT * from ${tableName}; + """ + + test { + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_4) + """ + exception "partition not exist" + } + + sql """ + alter table ${tableName} add PARTITION `p4` VALUES IN ('4') + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_4) + """ + + waitingMTMVTaskFinishedByMvName(mvName) + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} From b2d1778a07c81c657fc347b92f93b9a6f7704c09 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Fri, 13 Dec 2024 15:06:04 +0800 Subject: [PATCH 5/5] 1 --- .../trees/plans/commands/info/RefreshMTMVInfo.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java index 422567cf865832..94e9243306989a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java @@ -22,10 +22,12 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVRelatedTableIf; @@ -34,9 +36,11 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Set; @@ -82,9 +86,9 @@ public void analyze(ConnectContext ctx) { private void checkPartitionExist(MTMV mtmv) throws org.apache.doris.common.AnalysisException { MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); - // todo: Unified locking sequence - mtmv.readLock(); - relatedTable.readLock(); + List tables = Lists.newArrayList(mtmv, relatedTable); + tables.sort(Comparator.comparing(TableIf::getId)); + MetaLockUtils.readLockTables(tables); try { if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { throw new AnalysisException( @@ -103,8 +107,7 @@ private void checkPartitionExist(MTMV mtmv) throws org.apache.doris.common.Analy } } } finally { - relatedTable.readUnlock(); - mtmv.readUnlock(); + MetaLockUtils.readUnlockTables(tables); } }