diff --git a/be/src/common/config.h b/be/src/common/config.h index 4a8812478e0edb..31ea35a5dcb1e1 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -240,7 +240,8 @@ namespace config { // cumulative compaction policy: max delta file's size unit:B CONF_Int32(cumulative_compaction_check_interval_seconds, "10"); - CONF_Int64(cumulative_compaction_num_singleton_deltas, "5"); + CONF_Int64(min_cumulative_compaction_num_singleton_deltas, "5"); + CONF_Int64(max_cumulative_compaction_num_singleton_deltas, "1000"); CONF_Int32(cumulative_compaction_num_threads, "1"); CONF_Int32(cumulative_compaction_num_threads_per_disk, "1"); CONF_Int64(cumulative_compaction_budgeted_bytes, "104857600"); diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 35ddb8174734dc..6f8510a45c6a46 100755 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -85,7 +85,7 @@ OLAPStatus CumulativeCompaction::pick_rowsets_to_compact() { // So the ultimate singleton rowset is revserved. RowsetSharedPtr rowset = candidate_rowsets[i]; if (_tablet->version_for_delete_predicate(rowset->version())) { - if (transient_rowsets.size() > config::cumulative_compaction_num_singleton_deltas) { + if (transient_rowsets.size() > config::min_cumulative_compaction_num_singleton_deltas) { _input_rowsets = transient_rowsets; break; } @@ -93,10 +93,14 @@ OLAPStatus CumulativeCompaction::pick_rowsets_to_compact() { continue; } + if (transient_rowsets.size() >= config::max_cumulative_compaction_num_singleton_deltas) { + // the threshold of files to compacted one time + break; + } transient_rowsets.push_back(rowset); } - if (transient_rowsets.size() > config::cumulative_compaction_num_singleton_deltas) { + if (transient_rowsets.size() > config::min_cumulative_compaction_num_singleton_deltas) { _input_rowsets = transient_rowsets; }