diff --git a/be/src/common/config.h b/be/src/common/config.h index 42edb26fa7a10f..83dc17203a0ee9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -292,10 +292,10 @@ namespace config { // rowset will be not given to base compaction. The unit is m byte. CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64"); // The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than - // this size, size_based policy also does cumulative compaction. The unit is m byte. + // this size, size_based policy may not do to cumulative compaction. The unit is m byte. CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64"); - // cumulative compaction policy: max delta file's size unit:B + // cumulative compaction policy: min and max delta file's number CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5"); CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000"); CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600"); diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index c2c8755c694e6e..246098dadf4384 100755 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -44,7 +44,7 @@ OLAPStatus CumulativeCompaction::compact() { // 1.calculate cumulative point _tablet->calculate_cumulative_point(); TRACE("calculated cumulative point"); - LOG(INFO) << "after calculate, current cumulative point is " << _tablet->cumulative_layer_point() + VLOG(1) << "after calculate, current cumulative point is " << _tablet->cumulative_layer_point() << ", tablet=" << _tablet->full_name() ; // 2. pick rowsets to compact diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp index 2e28c58af57dec..22ebd9836caa6b 100644 --- a/be/src/olap/cumulative_compaction_policy.cpp +++ b/be/src/olap/cumulative_compaction_policy.cpp @@ -270,7 +270,7 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets( rs_iter = input_rowsets->erase(rs_iter); } - LOG(INFO) << "cumulative compaction size_based policy, compaction_score = " << *compaction_score + VLOG(1) << "cumulative compaction size_based policy, compaction_score = " << *compaction_score << ", total_size = " << total_size << ", calc promotion size value = " << promotion_size << ", tablet = " << tablet->full_name() << ", input_rowset size " @@ -467,4 +467,4 @@ void CumulativeCompactionPolicyFactory::_parse_cumulative_compaction_policy(std: *policy_type = NUM_BASED_POLICY; } } -} \ No newline at end of file +} diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 57ed54d58a75c0..d7dc38c851a355 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -351,6 +351,12 @@ void StorageEngine::_compaction_tasks_producer_callback() { for (const auto& tablet : tablets_compaction) { int64_t permits = tablet->calc_compaction_score(compaction_type); if (_permit_limiter.request(permits)) { + { + // Push to _tablet_submitted_compaction before submitting task + std::unique_lock lock(_tablet_submitted_compaction_mutex); + _tablet_submitted_compaction[tablet->data_dir()].emplace_back( + tablet->tablet_id()); + } if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) { _compaction_thread_pool->submit_func([=]() { CgroupsMgr::apply_system_cgroup(); @@ -386,9 +392,6 @@ void StorageEngine::_compaction_tasks_producer_callback() { } }); } - std::unique_lock lock(_tablet_submitted_compaction_mutex); - _tablet_submitted_compaction[tablet->data_dir()].emplace_back( - tablet->tablet_id()); } } } else { diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index bf7d09890dac89..a6ddd48d10e502 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -766,6 +766,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( VLOG(1) << "Found the best tablet for compaction. " << "compaction_type=" << compaction_type_str << ", tablet_id=" << best_tablet->tablet_id() + << ", path=" << data_dir->path() << ", highest_score=" << highest_score; // TODO(lingbin): Remove 'max' from metric name, it would be misunderstood as the // biggest in history(like peak), but it is really just the value at current moment.