From a8530ae6e9284183b91c35bfb20dbd2b21ee4fa2 Mon Sep 17 00:00:00 2001 From: garenshi Date: Thu, 19 Dec 2024 19:49:37 +0800 Subject: [PATCH 1/3] [fix](compaction) fix compaction producer hold for permits leak --- be/src/olap/olap_server.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 90d0883984e78b..e7b4f411bfa8b3 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -1073,6 +1073,9 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, "task, tablet_id=" << tablet->tablet_id() << ", tablet_state=" << tablet->tablet_state(); _pop_tablet_from_submitted_compaction(tablet, compaction_type); + if (!force) { + _permit_limiter.release(permits); + } return; } tablet->compaction_stage = CompactionStage::EXECUTING; From 3a9d13e5e0c175a606e06d2f8729219946b944ef Mon Sep 17 00:00:00 2001 From: garenshi Date: Tue, 31 Dec 2024 19:46:10 +0800 Subject: [PATCH 2/3] [Improve](mtmv) skip the generation of invalid task for refresh mtmv --- be/src/olap/olap_server.cpp | 3 - .../org/apache/doris/mtmv/MTMVService.java | 11 ++- .../data/mtmv_p0/test_commit_mtmv.out | 20 +++++ .../suites/mtmv_p0/test_commit_mtmv.groovy | 73 +++++++++++++++++++ 4 files changed, 103 insertions(+), 4 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index e7b4f411bfa8b3..90d0883984e78b 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -1073,9 +1073,6 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet, "task, tablet_id=" << tablet->tablet_id() << ", tablet_state=" << tablet->tablet_state(); _pop_tablet_from_submitted_compaction(tablet, compaction_type); - if (!force) { - _permit_limiter.release(permits); - } return; } tablet->compaction_stage = CompactionStage::EXECUTING; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java index 278811d3a991f7..26c6bfb10e9876 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java @@ -198,7 +198,7 @@ public void processEvent(Event event) throws EventException { try { // check if mtmv should trigger by event MTMV mtmv = (MTMV) MTMVUtil.getTable(baseTableInfo); - if (mtmv.getRefreshInfo().getRefreshTriggerInfo().getRefreshTrigger().equals(RefreshTrigger.COMMIT)) { + if (canRefresh(mtmv, table)) { jobManager.onCommit(mtmv); } } catch (Exception e) { @@ -206,4 +206,13 @@ public void processEvent(Event event) throws EventException { } } } + + private boolean canRefresh(MTMV mtmv, TableIf table) { + if (mtmv.getExcludedTriggerTables().contains(table.getName())) { + LOG.info("skip refresh mtmv: {}, because exclude trigger table: {}", + mtmv.getName(), table.getName()); + return false; + } + return mtmv.getRefreshInfo().getRefreshTriggerInfo().getRefreshTrigger().equals(RefreshTrigger.COMMIT); + } } diff --git a/regression-test/data/mtmv_p0/test_commit_mtmv.out b/regression-test/data/mtmv_p0/test_commit_mtmv.out index 208638b4c10a11..433d55ef4b8964 100644 --- a/regression-test/data/mtmv_p0/test_commit_mtmv.out +++ b/regression-test/data/mtmv_p0/test_commit_mtmv.out @@ -61,3 +61,23 @@ -- !mv1_replace -- 3 2017-03-15 3 +-- !mv_sag -- +1 1 60 + +-- !task_sag -- +{"triggerMode":"COMMIT","partitions":[],"isComplete":false} + +-- !mv_sag1 -- +1 1 60 + +-- !task_sag1 -- +{"triggerMode":"COMMIT","partitions":[],"isComplete":false} + +-- !mv_sag2 -- +1 1 60 +1 2 70 +2 1 70 + +-- !task_sag2 -- +{"triggerMode":"COMMIT","partitions":[],"isComplete":false} + diff --git a/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy b/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy index d8161a3fc922e0..05120ff7a9b055 100644 --- a/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy @@ -149,4 +149,77 @@ suite("test_commit_mtmv") { sql """drop materialized view if exists ${mvName2};""" sql """drop table if exists `${tableName}`""" + //===========test excluded_trigger_tables=========== + def tblStu = "test_commit_mtmv_tbl_stu" + def tblGrade = "test_commit_mtmv_tbl_grade" + def mvSag = "test_commit_mv_sag" + sql """drop materialized view if exists ${mvSag};""" + sql """drop table if exists `${tblStu}`""" + sql """drop table if exists `${tblGrade}`""" + sql """ + CREATE TABLE `${tblStu}` ( + `sid` int(32) NULL, + `sname` varchar(32) NULL, + ) ENGINE=OLAP + DUPLICATE KEY(`sid`) + DISTRIBUTED BY HASH(`sid`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + CREATE TABLE `${tblGrade}` ( + `sid` int(32) NULL, + `cid` int(32) NULL, + `score` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sid`) + DISTRIBUTED BY HASH(`sid`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvSag} + REFRESH COMPLETE ON commit + DISTRIBUTED BY HASH(`sid`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "excluded_trigger_tables" = "${tblGrade}" + ) + AS select a.sid,b.cid,b.score from ${tblStu} a join ${tblGrade} b on a.sid = b.sid; + """ + + sleep(1000) + + sql """ + insert into ${tblGrade} values(1, 1, 60); + insert into ${tblStu} values(1, 'sam'); + """ + def sagJobName = getJobName(dbName, mvSag); + waitingMTMVTaskFinished(sagJobName) + order_qt_mv_sag "SELECT * FROM ${mvSag} order by sid,cid" + order_qt_task_sag "SELECT TaskContext from tasks('type'='mv') where MvName='${mvSag}' order by CreateTime desc limit 1" + + sql """ + insert into ${tblGrade} values(1, 2, 70); + """ + waitingMTMVTaskFinished(sagJobName) + order_qt_mv_sag1 "SELECT * FROM ${mvSag} order by sid,cid" + order_qt_task_sag1 "SELECT TaskContext from tasks('type'='mv') where MvName='${mvSag}' order by CreateTime desc limit 1" + + sql """ + insert into ${tblGrade} values(2, 1, 70); + insert into ${tblStu} values(2, 'jack'); + """ + + waitingMTMVTaskFinished(sagJobName) + order_qt_mv_sag2 "SELECT * FROM ${mvSag} order by sid,cid" + order_qt_task_sag2 "SELECT TaskContext from tasks('type'='mv') where MvName='${mvSag}' order by CreateTime desc limit 1" + + sql """drop materialized view if exists ${mvSag};""" + sql """drop table if exists `${tblStu}`""" + sql """drop table if exists `${tblGrade}`""" } From fd5e04b6ba58c43932147b47fa672f427f19bc98 Mon Sep 17 00:00:00 2001 From: garenshi Date: Fri, 3 Jan 2025 10:40:39 +0800 Subject: [PATCH 3/3] 1 --- regression-test/suites/mtmv_p0/test_commit_mtmv.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy b/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy index 05120ff7a9b055..bb4c3f8f7cefd3 100644 --- a/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_commit_mtmv.groovy @@ -183,6 +183,7 @@ suite("test_commit_mtmv") { sql """ CREATE MATERIALIZED VIEW ${mvSag} + BUILD DEFERRED REFRESH COMPLETE ON commit DISTRIBUTED BY HASH(`sid`) BUCKETS 1 PROPERTIES ( @@ -192,8 +193,6 @@ suite("test_commit_mtmv") { AS select a.sid,b.cid,b.score from ${tblStu} a join ${tblGrade} b on a.sid = b.sid; """ - sleep(1000) - sql """ insert into ${tblGrade} values(1, 1, 60); insert into ${tblStu} values(1, 'sam');