Skip to content

Ensure affinity groups are honored when VMs are deployed in parallel #9201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DaanHoogland merged 1 commit into from
Aug 12, 2024
Merged

Ensure affinity groups are honored when VMs are deployed in parallel #9201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,7 @@ public interface AffinityGroupDao extends GenericDao<AffinityGroupVO, Long> {
AffinityGroupVO findByAccountAndType(Long accountId, String string);

AffinityGroupVO findDomainLevelGroupByType(Long domainId, String string);

List<AffinityGroupVO> listByIds(List<Long> ids, boolean exclusive);

}
12 changes: 12 additions & 0 deletions engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import com.cloud.utils.db.SearchCriteria;

public class AffinityGroupDaoImpl extends GenericDaoBase<AffinityGroupVO, Long> implements AffinityGroupDao {
private SearchBuilder<AffinityGroupVO> IdsSearch;
private SearchBuilder<AffinityGroupVO> AccountIdSearch;
private SearchBuilder<AffinityGroupVO> AccountIdNameSearch;
private SearchBuilder<AffinityGroupVO> AccountIdNamesSearch;
Expand All @@ -47,6 +48,10 @@ public AffinityGroupDaoImpl() {

@PostConstruct
protected void init() {
IdsSearch = createSearchBuilder();
IdsSearch.and("idIn", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
IdsSearch.done();

AccountIdSearch = createSearchBuilder();
AccountIdSearch.and("accountId", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
AccountIdSearch.done();
Expand Down Expand Up @@ -158,4 +163,11 @@ public AffinityGroupVO findDomainLevelGroupByType(Long domainId, String type) {
sc.setJoinParameters("domainTypeSearch", "domainId", domainId);
return findOneBy(sc);
}

@Override
public List<AffinityGroupVO> listByIds(List<Long> ids, boolean exclusive) {
SearchCriteria<AffinityGroupVO> sc = IdsSearch.create();
sc.setParameters("idIn", ids.toArray());
return lockRows(sc, null, exclusive);
}
}
39 changes: 29 additions & 10 deletions ...ors/host-affinity/src/main/java/org/apache/cloudstack/affinity/HostAffinityProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,14 @@
import java.util.Set;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.stream.Collectors;

import javax.inject.Inject;

import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;

Expand Down Expand Up @@ -56,9 +61,16 @@ public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, Exclud
VirtualMachine vm = vmProfile.getVirtualMachine();
List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
if (CollectionUtils.isNotEmpty(vmGroupMappings)) {
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
processAffinityGroup(vmGroupMapping, plan, vm, vmList);
}
List<Long> affinityGroupIdList = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
_affinityGroupDao.listByIds(affinityGroupIdList, true);
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
processAffinityGroup(vmGroupMapping, plan, vm, vmList);
}
}
});
}
}

Expand Down Expand Up @@ -132,16 +144,23 @@ public boolean check(VirtualMachineProfile vmProfile, DeployDestination plannedD
long plannedHostId = plannedDestination.getHost().getId();
VirtualMachine vm = vmProfile.getVirtualMachine();
List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
if (CollectionUtils.isEmpty(vmGroupMappings)) {
return true;
}
List<Long> affinityGroupIds = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
return Transaction.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
_affinityGroupDao.listByIds(affinityGroupIds, true);
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
if (!checkAffinityGroup(vmGroupMapping, vm, plannedHostId)) {
return false;
}

if (CollectionUtils.isNotEmpty(vmGroupMappings)) {
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
if (!checkAffinityGroup(vmGroupMapping, vm, plannedHostId)) {
return false;
}
return true;
}
}

return true;
});
}

/**
Expand Down
114 changes: 72 additions & 42 deletions ...anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,17 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.naming.ConfigurationException;

import com.cloud.utils.DateUtil;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;

import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
Expand All @@ -36,7 +43,6 @@
import com.cloud.deploy.DeploymentPlan;
import com.cloud.deploy.DeploymentPlanner.ExcludeList;
import com.cloud.exception.AffinityConflictException;
import com.cloud.utils.DateUtil;
import com.cloud.utils.NumbersUtil;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
Expand Down Expand Up @@ -67,40 +73,54 @@ public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, Exclud
VirtualMachine vm = vmProfile.getVirtualMachine();
List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());

for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
if (vmGroupMapping != null) {
AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());

if (s_logger.isDebugEnabled()) {
s_logger.debug("Processing affinity group " + group.getName() + " for VM Id: " + vm.getId());
if (CollectionUtils.isEmpty(vmGroupMappings)) {
return;
}
List<Long> affinityGroupIds = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
_affinityGroupDao.listByIds(affinityGroupIds, true);
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
processAffinityGroup(vmGroupMapping, avoid, vm);
}
}
});

List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
groupVMIds.remove(vm.getId());
}

for (Long groupVMId : groupVMIds) {
VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
if (groupVM != null && !groupVM.isRemoved()) {
if (groupVM.getHostId() != null) {
avoid.addHost(groupVM.getHostId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Added host " + groupVM.getHostId() + " to avoid set, since VM " + groupVM.getId() + " is present on the host");
}
} else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
avoid.addHost(groupVM.getLastHostId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Added host " + groupVM.getLastHostId() + " to avoid set, since VM " + groupVM.getId() +
" is present on the host, in Stopped state but has reserved capacity");
}
}
protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm) {
if (vmGroupMapping != null) {
AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());

if (s_logger.isDebugEnabled()) {
s_logger.debug("Processing affinity group " + group.getName() + " for VM Id: " + vm.getId());
}

List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
groupVMIds.remove(vm.getId());

for (Long groupVMId : groupVMIds) {
VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
if (groupVM != null && !groupVM.isRemoved()) {
if (groupVM.getHostId() != null) {
avoid.addHost(groupVM.getHostId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Added host " + groupVM.getHostId() + " to avoid set, since VM " + groupVM.getId() + " is present on the host");
}
}
} else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
avoid.addHost(groupVM.getLastHostId());
if (s_logger.isDebugEnabled()) {
s_logger.debug("Added host " + groupVM.getLastHostId() + " to avoid set, since VM " + groupVM.getId() +
" is present on the host, in Stopped state but has reserved capacity");
}
}
}
}
}

}

@Override
Expand All @@ -121,25 +141,35 @@ public boolean check(VirtualMachineProfile vmProfile, DeployDestination plannedD
VirtualMachine vm = vmProfile.getVirtualMachine();

List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
if (CollectionUtils.isEmpty(vmGroupMappings)) {
return true;
}

for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
// if more than 1 VM's are present in the group then check for
// conflict due to parallel deployment
List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(vmGroupMapping.getAffinityGroupId());
groupVMIds.remove(vm.getId());

for (Long groupVMId : groupVMIds) {
VMReservationVO vmReservation = _reservationDao.findByVmId(groupVMId);
if (vmReservation != null && vmReservation.getHostId() != null && vmReservation.getHostId().equals(plannedHostId)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Planned destination for VM " + vm.getId() + " conflicts with an existing VM " + vmReservation.getVmId() +
" reserved on the same host " + plannedHostId);
List<Long> affinityGroupIds = vmGroupMappings.stream().map(AffinityGroupVMMapVO::getAffinityGroupId).collect(Collectors.toList());
return Transaction.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
_affinityGroupDao.listByIds(affinityGroupIds, true);
for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
// if more than 1 VM's are present in the group then check for
// conflict due to parallel deployment
List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(vmGroupMapping.getAffinityGroupId());
groupVMIds.remove(vm.getId());

for (Long groupVMId : groupVMIds) {
VMReservationVO vmReservation = _reservationDao.findByVmId(groupVMId);
if (vmReservation != null && vmReservation.getHostId() != null && vmReservation.getHostId().equals(plannedHostId)) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Planned destination for VM " + vm.getId() + " conflicts with an existing VM " + vmReservation.getVmId() +
" reserved on the same host " + plannedHostId);
}
return false;
}
}
return false;
}
return true;
}
}
return true;
});
}

}