From 7315f6913b2895fb144e3052df2c5a874b192c34 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Wed, 22 Oct 2025 11:24:58 +0800 Subject: [PATCH] [chore](cold) add a score threshold for cold data compaction (#55333) --- be/src/common/config.cpp | 1 + be/src/common/config.h | 1 + be/src/olap/olap_server.cpp | 2 +- .../suites/cold_heat_separation/cold_data_compaction.groovy | 6 ++++++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 32683dc132edd8..9fedd42fc17945 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1019,6 +1019,7 @@ DEFINE_mInt32(remove_unused_remote_files_interval_sec, "21600"); // 6h DEFINE_mInt32(confirm_unused_remote_files_interval_sec, "60"); DEFINE_Int32(cold_data_compaction_thread_num, "2"); DEFINE_mInt32(cold_data_compaction_interval_sec, "1800"); +DEFINE_mInt32(cold_data_compaction_score_threshold, "100"); DEFINE_String(tmp_file_dir, "tmp"); diff --git a/be/src/common/config.h b/be/src/common/config.h index afba63bebe55a1..a3e2900715030b 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1045,6 +1045,7 @@ DECLARE_mInt32(remove_unused_remote_files_interval_sec); // 6h DECLARE_mInt32(confirm_unused_remote_files_interval_sec); DECLARE_Int32(cold_data_compaction_thread_num); DECLARE_mInt32(cold_data_compaction_interval_sec); +DECLARE_mInt32(cold_data_compaction_score_threshold); DECLARE_Int32(min_s3_file_system_thread_num); DECLARE_Int32(max_s3_file_system_thread_num); diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 906cffc9a4221a..149131bee3f5cf 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -1545,7 +1545,7 @@ void StorageEngine::_cold_data_compaction_producer_callback() { for (auto& t : tablets) { if (t->replica_id() == t->cooldown_conf_unlocked().cooldown_replica_id) { auto score = t->calc_cold_data_compaction_score(); - if (score < 4) { + if (score < config::cold_data_compaction_score_threshold) { continue; } tablet_to_compact.emplace_back(t, score); diff --git a/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy b/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy index d7bc2c2625c2f5..2a3f23c61674b2 100644 --- a/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy +++ b/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy @@ -37,6 +37,11 @@ suite("test_cold_data_compaction", "nonConcurrent") { } } + def custoBeConfig = [ + cold_data_compaction_score_threshold : 4 + ] + + setBeConfigTemporary(custoBeConfig) { String suffix = UUID.randomUUID().hashCode().abs().toString() String s3Prefix = "regression/cold_data_compaction" multi_sql """ @@ -152,4 +157,5 @@ suite("test_cold_data_compaction", "nonConcurrent") { logger.info("after drop t_recycle_in_s3, remote file number is ${filesAfterDrop.size()}") return filesAfterDrop.size() == 0 }) + } }