Add configuration to limit the number of rows deleted from vm_stats

engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java

@@ -75,8 +75,10 @@ public interface VmStatsDao extends GenericDao<VmStatsVO, Long> {
     /**
      * Removes (expunges) all VM stats with {@code timestamp} less than
      * a given Date.
-     * @param limit the maximum date to keep stored. Records that exceed this limit will be removed.
+     * @param limitDate the maximum date to keep stored. Records that exceed this limit will be removed.
+     * @param limitPerQuery the maximum amount of rows to be removed in a single query. We loop if there are still rows to be removed after a given query.
+     *                      If 0 or negative, no limit is used.
      */
-    void removeAllByTimestampLessThan(Date limit);
+    void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery);
 
 }

engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java

@@ -21,6 +21,7 @@
 
 import javax.annotation.PostConstruct;
 
+import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
 import com.cloud.utils.db.Filter;
@@ -33,6 +34,8 @@
 @Component
 public class VmStatsDaoImpl extends GenericDaoBase<VmStatsVO, Long> implements VmStatsDao {
 
+    protected Logger logger = Logger.getLogger(getClass());
+
     protected SearchBuilder<VmStatsVO> vmIdSearch;
     protected SearchBuilder<VmStatsVO> vmIdTimestampGreaterThanEqualSearch;
     protected SearchBuilder<VmStatsVO> vmIdTimestampLessThanEqualSearch;
@@ -113,10 +116,22 @@ public void removeAllByVmId(long vmId) {
     }
 
     @Override
-    public void removeAllByTimestampLessThan(Date limit) {
+    public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) {
         SearchCriteria<VmStatsVO> sc = timestampSearch.create();
-        sc.setParameters("timestamp", limit);
-        expunge(sc);
+        sc.setParameters("timestamp", limitDate);
+
+        logger.debug(String.format("Starting to remove all vm_stats rows older than [%s].", limitDate));
+
+        long totalRemoved = 0;
+        long removed;
+
+        do {
+            removed = expunge(sc, limitPerQuery);
+            totalRemoved += removed;
+            logger.trace(String.format("Removed [%s] vm_stats rows on the last update and a sum of [%s] vm_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
+        } while (limitPerQuery > 0 && removed >= limitPerQuery);
+
+        logger.info(String.format("Removed a total of [%s] vm_stats rows older than [%s].", totalRemoved, limitDate));
     }
 
 }

framework/db/src/main/java/com/cloud/utils/db/GenericDao.java

@@ -229,6 +229,14 @@ public interface GenericDao<T, ID extends Serializable> {
      */
     int expunge(final SearchCriteria<T> sc);
 
+    /**
+     * Delete the entity beans specified by the search criteria with a given limit
+     * @param sc Search criteria
+     * @param limit Maximum number of rows that will be affected
+     * @return Number of rows deleted
+     */
+    int expunge(SearchCriteria<T> sc, long limit);
+
     /**
      * expunge the removed rows.
      */

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -1226,9 +1226,14 @@ public boolean expunge(final ID id) {
         }
     }
 
-    // FIXME: Does not work for joins.
     @Override
     public int expunge(final SearchCriteria<T> sc) {
+        return expunge(sc, -1);
+    }
+
+    // FIXME: Does not work for joins.
+    @Override
+    public int expunge(final SearchCriteria<T> sc, long limit) {
         if (sc == null) {
             throw new CloudRuntimeException("Call to throw new expunge with null search Criteria");
         }
@@ -1241,6 +1246,11 @@ public int expunge(final SearchCriteria<T> sc) {
             str.append(sc.getWhereClause());
         }
 
+        if (limit > 0) {
+            str.append(" LIMIT ");
+            str.append(limit);
+        }
+
         final String sql = str.toString();
 
         final TransactionLegacy txn = TransactionLegacy.currentTxn();

server/src/main/java/com/cloud/server/StatsCollector.java

@@ -114,9 +114,6 @@
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
 import com.cloud.serializer.GsonHelper;
-import com.cloud.server.StatsCollector.AbstractStatsCollector;
-import com.cloud.server.StatsCollector.AutoScaleMonitor;
-import com.cloud.server.StatsCollector.StorageCollector;
 import com.cloud.storage.ImageStoreDetailsUtil;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage;
@@ -287,6 +284,11 @@ public String toString() {
     protected static ConfigKey<Boolean> vmStatsCollectUserVMOnly = new ConfigKey<>("Advanced", Boolean.class, "vm.stats.user.vm.only", "false",
             "When set to 'false' stats for system VMs will be collected otherwise stats collection will be done only for user VMs", true);
 
+    protected static ConfigKey<Long> vmStatsRemoveBatchSize = new ConfigKey<>("Advanced", Long.class, "vm.stats.remove.batch.size", "0", "Indicates the" +
+            " limit applied to delete vm_stats entries while running the clean-up task. With this, ACS will run the delete query, applying the limit, as many times as necessary" +
+            " to delete all entries older than the value defined in vm.stats.max.retention.time. This is advised when retaining several days of records, which can lead to slowness" +
+            " on the delete query. Zero (0) means that no limit will be applied, therefore, the query will run once and without limit, keeping the default behavior.", true);
+
     protected static ConfigKey<Boolean> vmDiskStatsRetentionEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.disk.stats.retention.enabled", "false",
             "When set to 'true' stats for VM disks will be stored in the database otherwise disk stats will not be stored", true);
 
@@ -1952,7 +1954,7 @@ protected void cleanUpVirtualMachineStats() {
         LOGGER.trace("Removing older VM stats records.");
         Date now = new Date();
         Date limit = DateUtils.addMinutes(now, -maxRetentionTime);
-        vmStatsDao.removeAllByTimestampLessThan(limit);
+        vmStatsDao.removeAllByTimestampLessThan(limit, vmStatsRemoveBatchSize.value());
     }
 
     /**
@@ -2126,7 +2128,7 @@ public String getConfigComponentName() {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri,
+        return new ConfigKey<?>[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri, vmStatsRemoveBatchSize,
             vmStatsIncrementMetrics, vmStatsMaxRetentionTime, vmStatsCollectUserVMOnly, vmDiskStatsRetentionEnabled, vmDiskStatsMaxRetentionTime,
                 MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL,
                 DATABASE_SERVER_STATUS_COLLECTION_INTERVAL,

server/src/test/java/com/cloud/server/StatsCollectorTest.java

@@ -28,7 +28,7 @@
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.VmStats;
 import com.cloud.vm.VmStatsVO;
-import com.cloud.vm.dao.VmStatsDao;
+import com.cloud.vm.dao.VmStatsDaoImpl;
 import com.google.gson.Gson;
 import com.tngtech.java.junit.dataprovider.DataProvider;
 import com.tngtech.java.junit.dataprovider.DataProviderRunner;
@@ -81,7 +81,7 @@ public class StatsCollectorTest {
     private static final String DEFAULT_DATABASE_NAME = "cloudstack";
 
     @Mock
-    VmStatsDao vmStatsDaoMock = Mockito.mock(VmStatsDao.class);
+    VmStatsDaoImpl vmStatsDaoMock;
 
     @Mock
     VmStatsEntry statsForCurrentIterationMock;
@@ -304,7 +304,7 @@ public void cleanUpVirtualMachineStatsTestIsDisabled() {
 
         statsCollector.cleanUpVirtualMachineStats();
 
-        Mockito.verify(vmStatsDaoMock, Mockito.never()).removeAllByTimestampLessThan(Mockito.any());
+        Mockito.verify(vmStatsDaoMock, Mockito.never()).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong());
     }
 
     @Test
@@ -313,7 +313,7 @@ public void cleanUpVirtualMachineStatsTestIsEnabled() {
 
         statsCollector.cleanUpVirtualMachineStats();
 
-        Mockito.verify(vmStatsDaoMock).removeAllByTimestampLessThan(Mockito.any());
+        Mockito.verify(vmStatsDaoMock).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong());
     }
 
     @Test

server/src/test/java/com/cloud/user/MockUsageEventDao.java

@@ -210,6 +210,11 @@ public int expunge(SearchCriteria<UsageEventVO> sc) {
         return 0;
     }
 
+    @Override
+    public int expunge(SearchCriteria<UsageEventVO> sc, long limit) {
+        return 0;
+    }
+
     @Override
     public void expunge() {