From f2df884a4439a30e870096dcf0203a75823df454 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Wed, 11 Jun 2025 20:43:42 +0800 Subject: [PATCH 1/4] 1 --- be/src/cloud/cloud_tablet_mgr.cpp | 13 +++++++------ be/src/cloud/config.cpp | 7 ++++--- be/src/cloud/config.h | 7 ++++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 376a935335f5d8..c3fa37e4519617 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -387,18 +387,19 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( using namespace std::chrono; auto now = duration_cast(system_clock::now().time_since_epoch()).count(); auto skip = [now, compaction_type](CloudTablet* t) { + int32_t max_version_config = t->max_version_config(); if (compaction_type == CompactionType::BASE_COMPACTION) { - return now - t->last_base_compaction_success_time_ms < config::base_compaction_freeze_interval_s * 1000 || - now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms; + return now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms || + (now - t->last_load_time_ms > config::base_compaction_load_max_freeze_interval_s * 1000 + && now - t->last_base_compaction_success_time_ms < config::base_compaction_success_min_freeze_interval_s * 1000 + && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); } // If tablet has too many rowsets but not be compacted for a long time, compaction should be performed // regardless of whether there is a load job recently. - - int32_t max_version_config = t->max_version_config(); return now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms || now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms || - (now - t->last_load_time_ms > config::cu_compaction_freeze_interval_s * 1000 - && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_interval_s * 1000 + (now - t->last_load_time_ms > config::cumu_compaction_load_max_freeze_interval_s * 1000 + && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_success_min_freeze_interval_s * 1000 && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); }; // We don't schedule tablets that are disabled for compaction diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index b0f80835598f1d..b53719440fcc94 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -43,9 +43,10 @@ DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); -DEFINE_mInt64(base_compaction_freeze_interval_s, "7200"); -DEFINE_mInt64(cu_compaction_freeze_interval_s, "1200"); -DEFINE_mInt64(cumu_compaction_interval_s, "1800"); +DEFINE_mInt64(base_compaction_load_max_freeze_interval_s, "4800"); +DEFINE_mInt64(base_compaction_success_min_freeze_interval_s, "7200"); +DEFINE_mInt64(cumu_compaction_load_max_freeze_interval_s, "1200"); +DEFINE_mInt64(cumu_compaction_success_min_freeze_interval_s, "1800"); DEFINE_mInt32(compaction_timeout_seconds, "86400"); DEFINE_mInt32(lease_compaction_interval_seconds, "20"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index f7b85231cbdb86..ceb524078f53a1 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -77,9 +77,10 @@ DECLARE_mInt32(sync_rowsets_slow_threshold_ms); DECLARE_mInt64(min_compaction_failure_interval_ms); DECLARE_mBool(enable_new_tablet_do_compaction); // For cloud read/write separate mode -DECLARE_mInt64(base_compaction_freeze_interval_s); -DECLARE_mInt64(cu_compaction_freeze_interval_s); -DECLARE_mInt64(cumu_compaction_interval_s); +DECLARE_mInt64(base_compaction_load_max_freeze_interval_s); +DECLARE_mInt64(base_compaction_success_min_freeze_interval_s); +DECLARE_mInt64(cumu_compaction_load_max_freeze_interval_s); +DECLARE_mInt64(cumu_compaction_success_min_freeze_interval_s); DECLARE_mInt32(compaction_timeout_seconds); DECLARE_mInt32(lease_compaction_interval_seconds); From be626245fc72c1f02939421c7b3185b56ba94e53 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Wed, 11 Jun 2025 21:06:42 +0800 Subject: [PATCH 2/4] 2 --- be/src/cloud/cloud_tablet_mgr.cpp | 17 ++++++++++------- be/src/cloud/config.cpp | 7 +++---- be/src/cloud/config.h | 7 +++---- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index c3fa37e4519617..9a59cbedeedfe2 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -389,18 +389,21 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( auto skip = [now, compaction_type](CloudTablet* t) { int32_t max_version_config = t->max_version_config(); if (compaction_type == CompactionType::BASE_COMPACTION) { - return now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms || - (now - t->last_load_time_ms > config::base_compaction_load_max_freeze_interval_s * 1000 - && now - t->last_base_compaction_success_time_ms < config::base_compaction_success_min_freeze_interval_s * 1000 + bool is_recent_failure = now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms; + bool is_frozen = (now - t->last_load_time_ms > config::compaction_load_max_freeze_interval_s * 1000 + && now - t->last_base_compaction_success_time_ms < config::base_compaction_freeze_interval_s * 1000 && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); + return is_recent_failure || is_frozen; } + // If tablet has too many rowsets but not be compacted for a long time, compaction should be performed // regardless of whether there is a load job recently. - return now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms || - now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms || - (now - t->last_load_time_ms > config::cumu_compaction_load_max_freeze_interval_s * 1000 - && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_success_min_freeze_interval_s * 1000 + bool is_recent_failure = now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms; + bool is_recent_no_suitable_version = now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms; + bool is_frozen = (now - t->last_load_time_ms > config::compaction_load_max_freeze_interval_s * 1000 + && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_interval_s * 1000 && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); + return is_recent_failure || is_recent_no_suitable_version || is_frozen; }; // We don't schedule tablets that are disabled for compaction auto disable = [](CloudTablet* t) { return t->tablet_meta()->tablet_schema()->disable_auto_compaction(); }; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index b53719440fcc94..6899542e364cab 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -43,10 +43,9 @@ DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt32(sync_rowsets_slow_threshold_ms, "1000"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); -DEFINE_mInt64(base_compaction_load_max_freeze_interval_s, "4800"); -DEFINE_mInt64(base_compaction_success_min_freeze_interval_s, "7200"); -DEFINE_mInt64(cumu_compaction_load_max_freeze_interval_s, "1200"); -DEFINE_mInt64(cumu_compaction_success_min_freeze_interval_s, "1800"); +DEFINE_mInt64(base_compaction_freeze_interval_s, "7200"); +DEFINE_mInt64(compaction_load_max_freeze_interval_s, "1200"); +DEFINE_mInt64(cumu_compaction_interval_s, "1800"); DEFINE_mInt32(compaction_timeout_seconds, "86400"); DEFINE_mInt32(lease_compaction_interval_seconds, "20"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index ceb524078f53a1..fd28e2446594cf 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -77,10 +77,9 @@ DECLARE_mInt32(sync_rowsets_slow_threshold_ms); DECLARE_mInt64(min_compaction_failure_interval_ms); DECLARE_mBool(enable_new_tablet_do_compaction); // For cloud read/write separate mode -DECLARE_mInt64(base_compaction_load_max_freeze_interval_s); -DECLARE_mInt64(base_compaction_success_min_freeze_interval_s); -DECLARE_mInt64(cumu_compaction_load_max_freeze_interval_s); -DECLARE_mInt64(cumu_compaction_success_min_freeze_interval_s); +DECLARE_mInt64(base_compaction_freeze_interval_s); +DECLARE_mInt64(compaction_load_max_freeze_interval_s); +DECLARE_mInt64(cumu_compaction_interval_s); DECLARE_mInt32(compaction_timeout_seconds); DECLARE_mInt32(lease_compaction_interval_seconds); From b6b5cc45c227f9ce6c6f6e6f5e6611686ee74fd5 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Thu, 19 Jun 2025 23:16:09 +0800 Subject: [PATCH 3/4] 3 --- be/src/cloud/cloud_tablet_mgr.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 9a59cbedeedfe2..9d56244f220725 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -31,6 +31,10 @@ namespace doris { uint64_t g_tablet_report_inactive_duration_ms = 0; +bvar::Adder g_base_compaction_not_frozen_tablet_num( + "g_base_compaction_not_frozen_tablet_num"); +bvar::Adder g_cumu_compaction_not_frozen_tablet_num( + "g_cumu_compaction_not_frozen_tablet_num"); namespace { // port from @@ -393,6 +397,7 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( bool is_frozen = (now - t->last_load_time_ms > config::compaction_load_max_freeze_interval_s * 1000 && now - t->last_base_compaction_success_time_ms < config::base_compaction_freeze_interval_s * 1000 && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); + g_base_compaction_not_frozen_tablet_num << !is_frozen; return is_recent_failure || is_frozen; } @@ -403,6 +408,7 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( bool is_frozen = (now - t->last_load_time_ms > config::compaction_load_max_freeze_interval_s * 1000 && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_interval_s * 1000 && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); + g_cumu_compaction_not_frozen_tablet_num << !is_frozen; return is_recent_failure || is_recent_no_suitable_version || is_frozen; }; // We don't schedule tablets that are disabled for compaction From 1661b1f48cc8697a88aff2c0f4e184d25efdc4cd Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Tue, 24 Jun 2025 16:53:51 +0800 Subject: [PATCH 4/4] 4 --- be/src/cloud/cloud_tablet_mgr.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 9d56244f220725..758cc0f57efd03 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -32,9 +32,9 @@ namespace doris { uint64_t g_tablet_report_inactive_duration_ms = 0; bvar::Adder g_base_compaction_not_frozen_tablet_num( - "g_base_compaction_not_frozen_tablet_num"); + "base_compaction_not_frozen_tablet_num"); bvar::Adder g_cumu_compaction_not_frozen_tablet_num( - "g_cumu_compaction_not_frozen_tablet_num"); + "cumu_compaction_not_frozen_tablet_num"); namespace { // port from