diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
index ac1cbe9154cbe9..d27c618c9144e0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
@@ -127,7 +127,14 @@ private void genPublishTask(List<Long> allBackends, TransactionState transaction
             partitionCommitInfos.addAll(tableCommitInfo.getIdToPartitionCommitInfo().values());
 
             try {
-                beIdToBaseTabletIds.putAll(getBaseTabletIdsForEachBe(transactionState, tableCommitInfo));
+                Map<Long, Set<Long>> map = getBaseTabletIdsForEachBe(transactionState, tableCommitInfo);
+                map.forEach((beId, newSet) -> {
+                    beIdToBaseTabletIds.computeIfPresent(beId, (id, orgSet) -> {
+                        orgSet.addAll(newSet);
+                        return orgSet;
+                    });
+                    beIdToBaseTabletIds.putIfAbsent(beId, newSet);
+                });
             } catch (MetaNotFoundException e) {
                 LOG.warn("exception occur when trying to get rollup tablets info", e);
             }