diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp
index 979938d5adeb1c..c29ab062c4157d 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -427,8 +427,10 @@ void CumulativeCompactionPolicy::pick_candidate_rowsets(
     int64_t now = UnixSeconds();
     for (auto& it : rs_version_map) {
         // find all rowset version greater than cumulative_point and skip the create time in skip_window_sec
-        if (it.first.first >= cumulative_point &&
-            (it.second->creation_time() + skip_window_sec < now)) {
+        if (it.first.first >= cumulative_point
+            && ((it.second->creation_time() + skip_window_sec < now)
+            // this case means a rowset has been compacted before which is not a new published rowset, so it should participate compaction
+            || (it.first.first != it.first.second))) {
             candidate_rowsets->push_back(it.second);
         }
     }