From fa353b3300a1ac50553f229792258867863b4ff4 Mon Sep 17 00:00:00 2001 From: Luwei Date: Wed, 25 Jun 2025 21:03:33 +0800 Subject: [PATCH 1/3] [enhance](compaction) optimize mow base compaction parameters --- be/src/cloud/cloud_base_compaction.cpp | 6 +++++- be/src/cloud/config.cpp | 2 +- be/src/common/config.cpp | 1 + be/src/common/config.h | 1 + be/src/olap/base_compaction.cpp | 6 +++++- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp index 99b916b45e5ccd..c3f559c5ae193b 100644 --- a/be/src/cloud/cloud_base_compaction.cpp +++ b/be/src/cloud/cloud_base_compaction.cpp @@ -200,9 +200,13 @@ Status CloudBaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; + int64 max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() + ? config::mow_base_compaction_max_compaction_score + : config::base_compaction_max_compaction_score; while (rowset_cnt < _input_rowsets.size()) { score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score(); - if (score > config::base_compaction_max_compaction_score) { + if (score > max_compaction_score) { break; } } diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 471de6464ae805..f0555889370498 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -43,7 +43,7 @@ DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); -DEFINE_mInt64(base_compaction_freeze_interval_s, "7200"); +DEFINE_mInt64(base_compaction_freeze_interval_s, "1800"); DEFINE_mInt64(cu_compaction_freeze_interval_s, "1200"); DEFINE_mInt64(cumu_compaction_interval_s, "1800"); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index dd3be9f152016a..ca93bd452e76f2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -429,6 +429,7 @@ DEFINE_mInt32(max_single_replica_compaction_threads, "-1"); DEFINE_Bool(enable_base_compaction_idle_sched, "true"); DEFINE_mInt64(base_compaction_min_rowset_num, "5"); DEFINE_mInt64(base_compaction_max_compaction_score, "20"); +DEFINE_mInt64(mow_base_compaction_max_compaction_score, "200"); DEFINE_mDouble(base_compaction_min_data_ratio, "0.3"); DEFINE_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 150e4e602fb76e..8f5804da394283 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -477,6 +477,7 @@ DECLARE_mInt32(max_single_replica_compaction_threads); DECLARE_Bool(enable_base_compaction_idle_sched); DECLARE_mInt64(base_compaction_min_rowset_num); DECLARE_mInt64(base_compaction_max_compaction_score); +DECLARE_mInt64(mow_base_compaction_max_compaction_score); DECLARE_mDouble(base_compaction_min_data_ratio); DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes); diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index 59ef901f3533ec..ada38f42df912c 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -181,9 +181,13 @@ Status BaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; + int64 max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() + ? config::mow_base_compaction_max_compaction_score + : config::base_compaction_max_compaction_score; while (rowset_cnt < _input_rowsets.size()) { score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score(); - if (score > config::base_compaction_max_compaction_score) { + if (score > max_compaction_score) { break; } } From 062558c96b8cffd4d1b1df1eeab2541922a8915c Mon Sep 17 00:00:00 2001 From: Luwei Date: Sun, 29 Jun 2025 18:51:16 +0800 Subject: [PATCH 2/3] fix --- be/src/cloud/cloud_base_compaction.cpp | 2 +- be/src/olap/base_compaction.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp index c3f559c5ae193b..d1fe4bd8c87181 100644 --- a/be/src/cloud/cloud_base_compaction.cpp +++ b/be/src/cloud/cloud_base_compaction.cpp @@ -200,7 +200,7 @@ Status CloudBaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; - int64 max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && + int64_t max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write() ? config::mow_base_compaction_max_compaction_score : config::base_compaction_max_compaction_score; diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index ada38f42df912c..602c4fde03c1ae 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -181,7 +181,7 @@ Status BaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; - int64 max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && + int64_t max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write() ? config::mow_base_compaction_max_compaction_score : config::base_compaction_max_compaction_score; From 199db7c5eb2f75ab1c61b2c78b68ea0868c614f2 Mon Sep 17 00:00:00 2001 From: Luwei Date: Sun, 29 Jun 2025 19:07:53 +0800 Subject: [PATCH 3/3] fix --- be/src/cloud/cloud_base_compaction.cpp | 6 +++--- be/src/olap/base_compaction.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp index d1fe4bd8c87181..d749459f8422d6 100644 --- a/be/src/cloud/cloud_base_compaction.cpp +++ b/be/src/cloud/cloud_base_compaction.cpp @@ -201,9 +201,9 @@ Status CloudBaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; int64_t max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write() - ? config::mow_base_compaction_max_compaction_score - : config::base_compaction_max_compaction_score; + _tablet->enable_unique_key_merge_on_write() + ? config::mow_base_compaction_max_compaction_score + : config::base_compaction_max_compaction_score; while (rowset_cnt < _input_rowsets.size()) { score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score(); if (score > max_compaction_score) { diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index 602c4fde03c1ae..b8f0026229734b 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -182,9 +182,9 @@ Status BaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; int64_t max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write() - ? config::mow_base_compaction_max_compaction_score - : config::base_compaction_max_compaction_score; + _tablet->enable_unique_key_merge_on_write() + ? config::mow_base_compaction_max_compaction_score + : config::base_compaction_max_compaction_score; while (rowset_cnt < _input_rowsets.size()) { score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score(); if (score > max_compaction_score) {