From faa4e7b1be23a58ab4df59809ed494339327022d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Apr 2023 13:25:17 +0530 Subject: [PATCH 01/31] Live storage migration of volume in scaleIO within same storage scaleio cluster --- .../main/java/com/cloud/vm/VirtualMachineManagerImpl.java | 3 ++- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 8 ++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 1b3d914b27a7..19b1cd3a7476 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -212,6 +212,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateVO; @@ -2904,7 +2905,7 @@ protected Map buildMapUsingUserInformation(VirtualMachinePr * */ protected void executeManagedStorageChecksWhenTargetStoragePoolProvided(StoragePoolVO currentPool, VolumeVO volume, StoragePoolVO targetPool) { - if (!currentPool.isManaged()) { + if (!currentPool.isManaged() || currentPool.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { return; } if (currentPool.getId() == targetPool.getId()) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index cec789767479..15bb5fca60d2 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2983,11 +2983,6 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { throw new InvalidParameterValueException("Live Migration of GPU enabled VM is not supported"); } - StoragePoolVO storagePoolVO = _storagePoolDao.findById(vol.getPoolId()); - if (storagePoolVO.getPoolType() == Storage.StoragePoolType.PowerFlex) { - throw new InvalidParameterValueException("Migrate volume of a running VM is unsupported on storage pool type " + storagePoolVO.getPoolType()); - } - // Check if the underlying hypervisor supports storage motion. Long hostId = vm.getHostId(); if (hostId != null) { @@ -3002,7 +2997,8 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { liveMigrateVolume = capabilities.isStorageMotionSupported(); } - if (liveMigrateVolume && HypervisorType.KVM.equals(host.getHypervisorType())) { + StoragePoolVO storagePoolVO = _storagePoolDao.findById(vol.getPoolId()); + if (liveMigrateVolume && HypervisorType.KVM.equals(host.getHypervisorType()) && !storagePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { StoragePoolVO destinationStoragePoolVo = _storagePoolDao.findById(storagePoolId); if (isSourceOrDestNotOnStorPool(storagePoolVO, destinationStoragePoolVo)) { From a782f371ead7746d21ba1a8aaef8cb4bb60a7018 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 31 Jan 2023 13:40:00 +0530 Subject: [PATCH 02/31] Added migrate command --- .../LibvirtMigrateVolumeCommandWrapper.java | 1 + .../driver/ScaleIOPrimaryDataStoreDriver.java | 113 +++++++++++++++--- 2 files changed, 100 insertions(+), 14 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 311eb670e99f..688bcce48c33 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -43,6 +43,7 @@ public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper")); - String value = configDao.getValue(Config.CopyVolumeWait.key()); - int copyVolumeWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); + // Volume migration within same PowerFlex/ScaleIO cluster (with same System ID) + DataStore srcStore = srcData.getDataStore(); + DataStore destStore = destData.getDataStore(); + Answer answer = null; + try { + long srcPoolId = srcStore.getId(); + long destPoolId = destStore.getId(); + + final ScaleIOGatewayClient client = getScaleIOClient(srcPoolId); + final String srcVolumePath = ((VolumeInfo) srcData).getPath(); + final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); + final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); + final String destStoragePoolId = destStoragePool.getPath(); - CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), copyVolumeWait, VirtualMachineManager.ExecuteInSequence.value()); + String value = configDao.getValue(Config.MigrateWait.key()); + int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue())); + MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getId(), srcVolumePath, destStoragePool, ((VolumeInfo) srcData).getAttachedVmName(), ((VolumeInfo) srcData).getVolumeType(), waitInterval); - Answer answer = null; - boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData); - EndPoint ep = destHost != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(destHost) : selector.select(srcData, encryptionRequired); - if (ep == null) { - String errorMsg = String.format("No remote endpoint to send command, unable to find a valid endpoint. Requires encryption support: %s", encryptionRequired); - LOGGER.error(errorMsg); - answer = new Answer(cmd, false, errorMsg); - } else { - answer = ep.sendMessage(cmd); + long hostId = 0; + VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName()); + if (instance.getState().equals(VirtualMachine.State.Running)) { + hostId = instance.getHostId(); + } + if (hostId == 0) { + hostId = selector.select(srcData, true).getId(); + } + HostVO host = hostDao.findById(hostId); + if (host == null) { + throw new CloudRuntimeException("Found no hosts to run resize command on"); + } + EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host); + + answer = ep.sendMessage(migrateVolumeCommand); + boolean migrateStatus = answer.getResult(); + + if (migrateStatus) { + String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(), + destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); + boolean renamed = client.renameVolume(srcVolumeId, newVolumeName); + + if (srcData.getId() != destData.getId()) { + VolumeVO destVolume = volumeDao.findById(destData.getId()); + // Volume Id in the PowerFlex/ScaleIO pool remains the same after the migration + // Update PowerFlex volume name only after it is renamed, to maintain the consistency + if (renamed) { + String newVolumePath = ScaleIOUtil.updatedPathWithVolumeName(srcVolumeId, newVolumeName); + destVolume.set_iScsiName(newVolumePath); + destVolume.setPath(newVolumePath); + } else { + destVolume.set_iScsiName(srcVolumePath); + destVolume.setPath(srcVolumePath); + } + volumeDao.update(destData.getId(), destVolume); + + VolumeVO srcVolume = volumeDao.findById(srcData.getId()); + srcVolume.set_iScsiName(null); + srcVolume.setPath(null); + srcVolume.setFolder(null); + volumeDao.update(srcData.getId(), srcVolume); + } else { + // Live migrate volume + VolumeVO volume = volumeDao.findById(srcData.getId()); + Long oldPoolId = volume.getPoolId(); + volume.setPoolId(destPoolId); + volume.setLastPoolId(oldPoolId); + volumeDao.update(srcData.getId(), volume); + } + + List snapshots = snapshotDao.listByVolumeId(srcData.getId()); + if (CollectionUtils.isNotEmpty(snapshots)) { + for (SnapshotVO snapshot : snapshots) { + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + if (snapshotStore == null) { + continue; + } + + String snapshotVolumeId = ScaleIOUtil.getVolumePath(snapshotStore.getInstallPath()); + String newSnapshotName = String.format("%s-%s-%s-%s", ScaleIOUtil.SNAPSHOT_PREFIX, snapshot.getId(), + destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); + renamed = client.renameVolume(snapshotVolumeId, newSnapshotName); + + snapshotStore.setDataStoreId(destPoolId); + // Snapshot Id in the PowerFlex/ScaleIO pool remains the same after the migration + // Update PowerFlex snapshot name only after it is renamed, to maintain the consistency + if (renamed) { + snapshotStore.setInstallPath(ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, newSnapshotName)); + } + snapshotDataStoreDao.update(snapshotStore.getId(), snapshotStore); + } + } + + answer = new Answer(null, true, null); + } else { + String errorMsg = "Failed to migrate PowerFlex volume: " + srcData.getId() + " to storage pool " + destPoolId; + LOGGER.debug(errorMsg); + answer = new Answer(null, false, errorMsg); + } + } catch (Exception e) { + LOGGER.error("Failed to migrate PowerFlex volume: " + srcData.getId() + " due to: " + e.getMessage()); + answer = new Answer(null, false, e.getMessage()); } return answer; From 9d8d174913e08814f45adcb1d5d78e89681b82db Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 22 Feb 2023 11:04:19 +0530 Subject: [PATCH 03/31] Recent changes of migration across clusters --- .../LibvirtMigrateVolumeCommandWrapper.java | 121 +++++++++++++++++- .../client/ScaleIOGatewayClientImpl.java | 2 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 68 +++++++++- 3 files changed, 188 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 688bcce48c33..420df975838e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -23,19 +23,35 @@ import com.cloud.agent.api.storage.MigrateVolumeAnswer; import com.cloud.agent.api.storage.MigrateVolumeCommand; import com.cloud.agent.api.to.DiskTO; +import com.cloud.exception.InternalErrorException; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import java.io.File; +import java.util.List; import java.util.Map; import java.util.UUID; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; +import org.libvirt.TypedParameter; +import org.libvirt.LibvirtException; +import org.libvirt.event.BlockJobListener; +import org.libvirt.event.BlockJobStatus; +import org.libvirt.event.BlockJobType; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { @@ -44,13 +60,116 @@ public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper destDetails = command.getDestDetails(); + + final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath()); + LOGGER.info("HARI Source volume ID: "+ srcVolumeId); + + final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath()); + LOGGER.info("HARI destination volume ID: "+ destVolumeId); + + final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + LOGGER.info("HARI destination system ID: "+ destSystemId); + + + final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId; + final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName; + + Domain dm = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + Connect conn = libvirtUtilitiesHelper.getConnection(); + dm = libvirtComputingResource.getDomain(conn, vmName); + + if (dm == null) { + return new MigrateVolumeAnswer(command, false, + "Migrate volume failed due to can not find vm: " + vmName, null); + } + + DomainInfo.DomainState domainState = dm.getInfo().state ; + if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { + return new MigrateVolumeAnswer(command, false, + "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); + } + + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + final String domXml = dm.getXMLDesc(0); + parser.parseDomainXML(domXml); + LOGGER.info(String.format("VM [%s] with XML configuration [%s] will be migrated to host.", vmName, domXml)); + + List disks = parser.getDisks(); + LibvirtVMDef.DiskDef diskdef = null; + for (final LibvirtVMDef.DiskDef disk : disks) { + final String file = disk.getDiskPath(); + LOGGER.info("HARIIII : " + file); + if (file != null && file.contains(srcVolumeId)) { + diskdef = disk; + break; + } + } + if (diskdef == null) { + throw new InternalErrorException("disk: " + srcPath + " is not attached before"); + } + diskdef.setDiskPath(diskFilePath); + LOGGER.info("HARIIII Destination xml : " + diskdef.toString()); + dm.blockCopy(srcPath, diskdef.toString(), new TypedParameter[]{}, 0); + BlockJobListener listener = new BlockJobListener() { + @Override + public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { + + } + }; + + return new MigrateVolumeAnswer(command, true, null, destPath); + } catch (LibvirtException e) { + String msg = "Migrate volume failed due to " + e.toString(); + LOGGER.warn(msg, e); + return new MigrateVolumeAnswer(command, false, msg, null); + } catch (InternalErrorException e) { + throw new RuntimeException(e); + } finally { + if (dm != null) { + try { + dm.free(); + } catch (LibvirtException l) { + LOGGER.trace("Ignoring libvirt error.", l); + }; + } + } + } + private MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); PrimaryDataStoreTO srcPrimaryDataStore = (PrimaryDataStoreTO)srcVolumeObjectTO.getDataStore(); Map srcDetails = command.getSrcDetails(); - String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath(); VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java index ad1279b52218..61e190d5239c 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java @@ -761,7 +761,7 @@ public boolean migrateVolume(final String srcVolumeId, final String destPoolId, } String srcPoolId = volume.getStoragePoolId(); - LOG.debug("Migrating the volume: " + srcVolumeId + " on the src pool: " + srcPoolId + " to the dest pool: " + destPoolId + + LOG.info("Migrating the volume: " + srcVolumeId + " on the src pool: " + srcPoolId + " to the dest pool: " + destPoolId + " in the same PowerFlex cluster"); post("/instances/Volume::" + srcVolumeId + "/action/migrateVTree", diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index ad97e7e7269b..98b27939ca9c 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import javax.inject.Inject; @@ -42,6 +43,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.RemoteHostEndPoint; @@ -71,6 +73,7 @@ import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.host.Host; @@ -127,6 +130,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private HostDao hostDao; @Inject private VMInstanceDao vmInstanceDao; + @Inject + private VolumeService volumeService; public ScaleIOPrimaryDataStoreDriver() { @@ -768,10 +773,23 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); final String destStoragePoolId = destStoragePool.getPath(); + //CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); + //String destVolumePath = createAnswer.getData().getPath(); + final String destVolumeId = UUID.randomUUID().toString(); + final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), + srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); + String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName); + + VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO(); + destVolTO.setPath(destVolumePath); + + Map srcDetails = getVolumeDetails((VolumeInfo) srcData); + Map destDetails = getVolumeDetails((VolumeInfo) destData); String value = configDao.getValue(Config.MigrateWait.key()); int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue())); - MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getId(), srcVolumePath, destStoragePool, ((VolumeInfo) srcData).getAttachedVmName(), ((VolumeInfo) srcData).getVolumeType(), waitInterval); + MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getTO(), destVolTO, + srcDetails, destDetails, waitInterval); long hostId = 0; VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName()); @@ -860,6 +878,52 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost return answer; } + private Map getVolumeDetails(VolumeInfo volumeInfo) { + long storagePoolId = volumeInfo.getPoolId(); + StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePoolId); + + if (!storagePoolVO.isManaged()) { + return null; + } + + Map volumeDetails = new HashMap<>(); + + VolumeVO volumeVO = volumeDao.findById(volumeInfo.getId()); + + volumeDetails.put(DiskTO.STORAGE_HOST, storagePoolVO.getHostAddress()); + volumeDetails.put(DiskTO.STORAGE_PORT, String.valueOf(storagePoolVO.getPort())); + volumeDetails.put(DiskTO.IQN, volumeVO.get_iScsiName()); + volumeDetails.put(DiskTO.PROTOCOL_TYPE, (volumeVO.getPoolType() != null) ? volumeVO.getPoolType().toString() : null); + volumeDetails.put(StorageManager.STORAGE_POOL_DISK_WAIT.toString(), String.valueOf(StorageManager.STORAGE_POOL_DISK_WAIT.valueIn(storagePoolVO.getId()))); + + volumeDetails.put(DiskTO.VOLUME_SIZE, String.valueOf(volumeVO.getSize())); + volumeDetails.put(DiskTO.SCSI_NAA_DEVICE_ID, getVolumeProperty(volumeInfo.getId(), DiskTO.SCSI_NAA_DEVICE_ID)); + + ChapInfo chapInfo = volumeService.getChapInfo(volumeInfo, volumeInfo.getDataStore()); + + if (chapInfo != null) { + volumeDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, chapInfo.getInitiatorUsername()); + volumeDetails.put(DiskTO.CHAP_INITIATOR_SECRET, chapInfo.getInitiatorSecret()); + volumeDetails.put(DiskTO.CHAP_TARGET_USERNAME, chapInfo.getTargetUsername()); + volumeDetails.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret()); + } + + String systemId = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID).getValue(); + volumeDetails.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId); + + return volumeDetails; + } + + private String getVolumeProperty(long volumeId, String property) { + VolumeDetailVO volumeDetails = volumeDetailsDao.findDetail(volumeId, property); + + if (volumeDetails != null) { + return volumeDetails.getValue(); + } + + return null; + } + private Answer migrateVolume(DataObject srcData, DataObject destData) { // Volume migration within same PowerFlex/ScaleIO cluster (with same System ID) DataStore srcStore = srcData.getDataStore(); @@ -875,6 +939,8 @@ private Answer migrateVolume(DataObject srcData, DataObject destData) { final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); final String destStoragePoolId = destStoragePool.getPath(); int migrationTimeout = StorageManager.KvmStorageOfflineMigrationWait.value(); + LOGGER.info("HARI source volume " + srcVolumeId); + LOGGER.info("HARI destination volume " + destStoragePoolId); boolean migrateStatus = client.migrateVolume(srcVolumeId, destStoragePoolId, migrationTimeout); if (migrateStatus) { String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(), From 7af37ed9bb8450e51b09958d62cd92efa7deef41 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 23 Feb 2023 10:46:17 +0530 Subject: [PATCH 04/31] Fixed uuid --- .../resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java | 4 +--- .../datastore/driver/ScaleIOPrimaryDataStoreDriver.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 420df975838e..8ad52664751e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -147,12 +147,10 @@ public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobS }; return new MigrateVolumeAnswer(command, true, null, destPath); - } catch (LibvirtException e) { + } catch (Exception e) { String msg = "Migrate volume failed due to " + e.toString(); LOGGER.warn(msg, e); return new MigrateVolumeAnswer(command, false, msg, null); - } catch (InternalErrorException e) { - throw new RuntimeException(e); } finally { if (dm != null) { try { diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 98b27939ca9c..81abff9ff385 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -775,7 +775,7 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost final String destStoragePoolId = destStoragePool.getPath(); //CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); //String destVolumePath = createAnswer.getData().getPath(); - final String destVolumeId = UUID.randomUUID().toString(); + final String destVolumeId = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 16); final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName); From 9f4145e261cd51382372ff74f2846b86d590e024 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 6 Mar 2023 13:12:19 +0530 Subject: [PATCH 05/31] recent changes --- .../LibvirtMigrateVolumeCommandWrapper.java | 116 +++++++++++------- .../driver/ScaleIOPrimaryDataStoreDriver.java | 23 ++-- 2 files changed, 84 insertions(+), 55 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 8ad52664751e..6ff437cd44cc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -48,6 +48,7 @@ import org.libvirt.Domain; import org.libvirt.DomainInfo; import org.libvirt.TypedParameter; +import org.libvirt.TypedUlongParameter; import org.libvirt.LibvirtException; import org.libvirt.event.BlockJobListener; import org.libvirt.event.BlockJobStatus; @@ -59,8 +60,6 @@ public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper srcDetails = command.getSrcDetails(); + final String srcSystemId = srcDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - Map destDetails = command.getDestDetails(); + final String diskFileName = ScaleIOUtil.DISK_NAME_PREFIX + srcSystemId + "-" + srcVolumeId; + final String srcFilePath = ScaleIOUtil.DISK_PATH + File.separator + diskFileName; - final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath()); LOGGER.info("HARI Source volume ID: "+ srcVolumeId); + LOGGER.info("HARI source volume PATH: "+ srcFilePath); + LOGGER.info("HARI source system ID: "+ srcSystemId); + // Destination Details + VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); + String destPath = destVolumeObjectTO.getPath(); final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath()); - LOGGER.info("HARI destination volume ID: "+ destVolumeId); - + Map destDetails = command.getDestDetails(); final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - LOGGER.info("HARI destination system ID: "+ destSystemId); - final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId; final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName; + LOGGER.info("HARI destination volume ID: "+ destVolumeId); + LOGGER.info("HARI destination system ID: "+ destSystemId); + Domain dm = null; try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); Connect conn = libvirtUtilitiesHelper.getConnection(); dm = libvirtComputingResource.getDomain(conn, vmName); - if (dm == null) { - return new MigrateVolumeAnswer(command, false, - "Migrate volume failed due to can not find vm: " + vmName, null); + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to can not find vm: " + vmName, null); } DomainInfo.DomainState domainState = dm.getInfo().state ; if (domainState != DomainInfo.DomainState.VIR_DOMAIN_RUNNING) { - return new MigrateVolumeAnswer(command, false, - "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); + return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); } - final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); - final String domXml = dm.getXMLDesc(0); - parser.parseDomainXML(domXml); - LOGGER.info(String.format("VM [%s] with XML configuration [%s] will be migrated to host.", vmName, domXml)); - - List disks = parser.getDisks(); - LibvirtVMDef.DiskDef diskdef = null; - for (final LibvirtVMDef.DiskDef disk : disks) { - final String file = disk.getDiskPath(); - LOGGER.info("HARIIII : " + file); - if (file != null && file.contains(srcVolumeId)) { - diskdef = disk; - break; - } - } - if (diskdef == null) { - throw new InternalErrorException("disk: " + srcPath + " is not attached before"); - } - diskdef.setDiskPath(diskFilePath); - LOGGER.info("HARIIII Destination xml : " + diskdef.toString()); - dm.blockCopy(srcPath, diskdef.toString(), new TypedParameter[]{}, 0); + LibvirtVMDef.DiskDef diskdef = generateDestinationDiskDefinition(dm, srcVolumeId, srcPath, diskFilePath); + + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); + TypedParameter[] parameters = new TypedParameter[1]; + parameters[0] = parameter; + LOGGER.info("Krishna source disk label: " + diskdef.getDiskLabel()); + + Domain finalDm = dm; + final Boolean[] copyStatus = {true}; + dm.blockCopy(diskdef.getDiskLabel(), diskdef.toString(), parameters, Domain.BlockCopyFlags.REUSE_EXT); BlockJobListener listener = new BlockJobListener() { @Override public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { - + if (type == BlockJobType.COPY && status == BlockJobStatus.READY) { + try { + finalDm.blockJobAbort(diskFilePath, 0); + copyStatus[0] = false; + } catch (LibvirtException e) { + throw new RuntimeException(e); + } + } } }; + dm.addBlockJobListener(listener); + while (copyStatus[0]) { + LOGGER.info("Waiting for the block copy to complete"); + } + + if (copyStatus[0]) { + String msg = "Migrate volume failed due to timeout"; + LOGGER.warn(msg); + return new MigrateVolumeAnswer(command, false, msg, null); + } return new MigrateVolumeAnswer(command, true, null, destPath); } catch (Exception e) { @@ -161,6 +166,31 @@ public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobS } } } + + private LibvirtVMDef.DiskDef generateDestinationDiskDefinition(Domain dm, String srcVolumeId, String srcPath, String diskFilePath) throws InternalErrorException, LibvirtException { + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + final String domXml = dm.getXMLDesc(0); + parser.parseDomainXML(domXml); + + List disks = parser.getDisks(); + LibvirtVMDef.DiskDef diskdef = null; + for (final LibvirtVMDef.DiskDef disk : disks) { + final String file = disk.getDiskPath(); + LOGGER.info("HARIIII disk: " + file); + if (file != null && file.contains(srcVolumeId)) { + diskdef = disk; + break; + } + } + if (diskdef == null) { + throw new InternalErrorException("disk: " + srcPath + " is not attached before"); + } + diskdef.setDiskPath(diskFilePath); + LOGGER.info("HARIIII Destination xml : " + diskdef.toString()); + + return diskdef; + } + private MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 81abff9ff385..6c3dd0f1b080 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -19,7 +19,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import javax.inject.Inject; @@ -773,18 +772,17 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); final String destStoragePoolId = destStoragePool.getPath(); - //CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); - //String destVolumePath = createAnswer.getData().getPath(); - final String destVolumeId = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 16); - final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), - srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); - String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName); + CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); + String destVolumePath = createAnswer.getData().getPath(); + //final String destVolumeId = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 16); + //final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); + //String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName); VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO(); destVolTO.setPath(destVolumePath); - Map srcDetails = getVolumeDetails((VolumeInfo) srcData); - Map destDetails = getVolumeDetails((VolumeInfo) destData); + Map srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore); + Map destDetails = getVolumeDetails((VolumeInfo) destData, destStore); String value = configDao.getValue(Config.MigrateWait.key()); int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue())); @@ -804,6 +802,7 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost throw new CloudRuntimeException("Found no hosts to run resize command on"); } EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host); + grantAccess(destData, ep, destData.getDataStore()); answer = ep.sendMessage(migrateVolumeCommand); boolean migrateStatus = answer.getResult(); @@ -878,8 +877,8 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost return answer; } - private Map getVolumeDetails(VolumeInfo volumeInfo) { - long storagePoolId = volumeInfo.getPoolId(); + private Map getVolumeDetails(VolumeInfo volumeInfo, DataStore dataStore) { + long storagePoolId = dataStore.getId(); StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePoolId); if (!storagePoolVO.isManaged()) { @@ -899,7 +898,7 @@ private Map getVolumeDetails(VolumeInfo volumeInfo) { volumeDetails.put(DiskTO.VOLUME_SIZE, String.valueOf(volumeVO.getSize())); volumeDetails.put(DiskTO.SCSI_NAA_DEVICE_ID, getVolumeProperty(volumeInfo.getId(), DiskTO.SCSI_NAA_DEVICE_ID)); - ChapInfo chapInfo = volumeService.getChapInfo(volumeInfo, volumeInfo.getDataStore()); + ChapInfo chapInfo = volumeService.getChapInfo(volumeInfo, dataStore); if (chapInfo != null) { volumeDetails.put(DiskTO.CHAP_INITIATOR_USERNAME, chapInfo.getInitiatorUsername()); From e5cbf31d087356083d5f61fca64d45cfa4196930 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 7 Mar 2023 17:37:03 +0530 Subject: [PATCH 06/31] Pivot changes --- .../wrapper/LibvirtMigrateVolumeCommandWrapper.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 6ff437cd44cc..b6930683069a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -39,6 +39,7 @@ import java.util.UUID; import com.cloud.storage.Storage; +import com.sun.jna.Pointer; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -53,6 +54,9 @@ import org.libvirt.event.BlockJobListener; import org.libvirt.event.BlockJobStatus; import org.libvirt.event.BlockJobType; +import org.libvirt.jna.ConnectionPointer; +import org.libvirt.jna.DomainPointer; +import org.libvirt.jna.Libvirt; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { @@ -122,7 +126,6 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); TypedParameter[] parameters = new TypedParameter[1]; parameters[0] = parameter; - LOGGER.info("Krishna source disk label: " + diskdef.getDiskLabel()); Domain finalDm = dm; final Boolean[] copyStatus = {true}; @@ -132,7 +135,7 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { if (type == BlockJobType.COPY && status == BlockJobStatus.READY) { try { - finalDm.blockJobAbort(diskFilePath, 0); + finalDm.blockJobAbort(diskFilePath, Domain.BlockJobAbortFlags.PIVOT); copyStatus[0] = false; } catch (LibvirtException e) { throw new RuntimeException(e); From 531a58235801475e71ee6bdfb79770a1a72742df Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 10 Mar 2023 17:17:00 +0530 Subject: [PATCH 07/31] working blockcopy api in libvirt --- .../LibvirtMigrateVolumeCommandWrapper.java | 9 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 238 +++++++++++------- 2 files changed, 152 insertions(+), 95 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index b6930683069a..c78ea453250b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -39,7 +39,6 @@ import java.util.UUID; import com.cloud.storage.Storage; -import com.sun.jna.Pointer; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -54,9 +53,6 @@ import org.libvirt.event.BlockJobListener; import org.libvirt.event.BlockJobStatus; import org.libvirt.event.BlockJobType; -import org.libvirt.jna.ConnectionPointer; -import org.libvirt.jna.DomainPointer; -import org.libvirt.jna.Libvirt; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { @@ -121,6 +117,11 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co return new MigrateVolumeAnswer(command, false, "Migrate volume failed due to VM is not running: " + vmName + " with domainState = " + domainState, null); } + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + PrimaryDataStoreTO spool = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); + KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid()); + pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); + LibvirtVMDef.DiskDef diskdef = generateDestinationDiskDefinition(dm, srcVolumeId, srcPath, diskFilePath); TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 6c3dd0f1b080..b70ae125f805 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -707,6 +707,9 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As } result = new CopyCommandResult(null, answer); + if (answer != null && !answer.getResult()) { + result.setResult(answer.getDetails()); + } callback.complete(result); } @@ -760,121 +763,176 @@ private Answer copyTemplateToVolume(DataObject srcData, DataObject destData, Hos private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost) { // Volume migration within same PowerFlex/ScaleIO cluster (with same System ID) + final long srcVolumeId = srcData.getId(); DataStore srcStore = srcData.getDataStore(); + Map srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore); + DataStore destStore = destData.getDataStore(); + final long destPoolId = destStore.getId(); + Map destDetails = getVolumeDetails((VolumeInfo) destData, destStore); + VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO(); + String destVolumePath = null; + + Host host = findEndpointForVolumeOperation(srcData); + EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host); + Answer answer = null; try { - long srcPoolId = srcStore.getId(); - long destPoolId = destStore.getId(); - - final ScaleIOGatewayClient client = getScaleIOClient(srcPoolId); - final String srcVolumePath = ((VolumeInfo) srcData).getPath(); - final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); - final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); - final String destStoragePoolId = destStoragePool.getPath(); CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); - String destVolumePath = createAnswer.getData().getPath(); - //final String destVolumeId = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 16); - //final String destScaleIOVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, srcData.getId(), srcData.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); - //String destVolumePath = String.format("%s:%s", destVolumeId, destScaleIOVolumeName); - - VolumeObjectTO destVolTO = (VolumeObjectTO) destData.getTO(); + destVolumePath = createAnswer.getData().getPath(); destVolTO.setPath(destVolumePath); - Map srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore); - Map destDetails = getVolumeDetails((VolumeInfo) destData, destStore); + grantAccess(destData, host, destData.getDataStore()); - String value = configDao.getValue(Config.MigrateWait.key()); - int waitInterval = NumbersUtil.parseInt(value, Integer.parseInt(Config.MigrateWait.getDefaultValue())); + int waitInterval = NumbersUtil.parseInt(configDao.getValue(Config.MigrateWait.key()), Integer.parseInt(Config.MigrateWait.getDefaultValue())); MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcData.getTO(), destVolTO, srcDetails, destDetails, waitInterval); - - long hostId = 0; - VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName()); - if (instance.getState().equals(VirtualMachine.State.Running)) { - hostId = instance.getHostId(); - } - if (hostId == 0) { - hostId = selector.select(srcData, true).getId(); - } - HostVO host = hostDao.findById(hostId); - if (host == null) { - throw new CloudRuntimeException("Found no hosts to run resize command on"); - } - EndPoint ep = RemoteHostEndPoint.getHypervisorHostEndPoint(host); - grantAccess(destData, ep, destData.getDataStore()); - answer = ep.sendMessage(migrateVolumeCommand); boolean migrateStatus = answer.getResult(); if (migrateStatus) { - String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(), - destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); - boolean renamed = client.renameVolume(srcVolumeId, newVolumeName); + updateVolumeAfterCopyVolume(srcData, destData); + updateSnapshotsAfterCopyVolume(srcData, destData); - if (srcData.getId() != destData.getId()) { - VolumeVO destVolume = volumeDao.findById(destData.getId()); - // Volume Id in the PowerFlex/ScaleIO pool remains the same after the migration - // Update PowerFlex volume name only after it is renamed, to maintain the consistency - if (renamed) { - String newVolumePath = ScaleIOUtil.updatedPathWithVolumeName(srcVolumeId, newVolumeName); - destVolume.set_iScsiName(newVolumePath); - destVolume.setPath(newVolumePath); - } else { - destVolume.set_iScsiName(srcVolumePath); - destVolume.setPath(srcVolumePath); - } - volumeDao.update(destData.getId(), destVolume); + LOGGER.debug(String.format("Successfully migrated migrate PowerFlex volume %d to storage pool %d", srcVolumeId, destPoolId)); + answer = new Answer(null, true, null); + } else { + String errorMsg = "Failed to migrate PowerFlex volume: " + srcVolumeId + " to storage pool " + destPoolId; + LOGGER.debug(errorMsg); + answer = new Answer(null, false, errorMsg); + } + } catch (Exception e) { + LOGGER.error("Failed to migrate PowerFlex volume: " + srcVolumeId + " due to: " + e.getMessage()); + answer = new Answer(null, false, e.getMessage()); + } - VolumeVO srcVolume = volumeDao.findById(srcData.getId()); - srcVolume.set_iScsiName(null); - srcVolume.setPath(null); - srcVolume.setFolder(null); - volumeDao.update(srcData.getId(), srcVolume); - } else { - // Live migrate volume - VolumeVO volume = volumeDao.findById(srcData.getId()); - Long oldPoolId = volume.getPoolId(); - volume.setPoolId(destPoolId); - volume.setLastPoolId(oldPoolId); - volumeDao.update(srcData.getId(), volume); + if (destVolumePath != null && !answer.getResult()) { + revertCopyVolumeOperations(srcData, destData, host, destVolumePath); + } + + return answer; + } + + private void checkForDestinationVolumeExistence(DataStore destStore, String destVolumePath) throws Exception { + int retryCount = 3; + while (retryCount > 0) { + try { + Thread.sleep(3000); // Try after few secs + String destScaleIOVolumeId = ScaleIOUtil.getVolumePath(destVolumePath); + final ScaleIOGatewayClient destClient = getScaleIOClient(destStore.getId()); + org.apache.cloudstack.storage.datastore.api.Volume destScaleIOVolume = destClient.getVolume(destScaleIOVolumeId); + if (destScaleIOVolume != null) { + return; } + } catch (Exception ex) { + LOGGER.error("Exception while checking for existence of the volume at " + destVolumePath + " - " + ex.getLocalizedMessage()); + throw ex; + } finally { + retryCount--; + } + } + } - List snapshots = snapshotDao.listByVolumeId(srcData.getId()); - if (CollectionUtils.isNotEmpty(snapshots)) { - for (SnapshotVO snapshot : snapshots) { - SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); - if (snapshotStore == null) { - continue; - } + private void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData) { + // destination volume is already created and volume path is set in database by this time at "CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());" + final long srcVolumeId = srcData.getId(); + final long destVolumeId = destData.getId(); + + if (srcVolumeId != destVolumeId) { + VolumeVO srcVolume = volumeDao.findById(srcVolumeId); + srcVolume.set_iScsiName(null); + srcVolume.setPath(null); + srcVolume.setFolder(null); + volumeDao.update(srcVolumeId, srcVolume); + } else { + // Live migrate volume + VolumeVO volume = volumeDao.findById(srcVolumeId); + Long oldPoolId = volume.getPoolId(); + volume.setLastPoolId(oldPoolId); + volumeDao.update(srcVolumeId, volume); + } + } + private Host findEndpointForVolumeOperation(DataObject srcData) { + long hostId = 0; + VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName()); + if (instance.getState().equals(VirtualMachine.State.Running)) { + hostId = instance.getHostId(); + } + if (hostId == 0) { + hostId = selector.select(srcData, true).getId(); + } + HostVO host = hostDao.findById(hostId); + if (host == null) { + throw new CloudRuntimeException("Found no hosts to run migrate volume command on"); + } - String snapshotVolumeId = ScaleIOUtil.getVolumePath(snapshotStore.getInstallPath()); - String newSnapshotName = String.format("%s-%s-%s-%s", ScaleIOUtil.SNAPSHOT_PREFIX, snapshot.getId(), - destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); - renamed = client.renameVolume(snapshotVolumeId, newSnapshotName); + return host; + } + private void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception { + final long srcVolumeId = srcData.getId(); + DataStore srcStore = srcData.getDataStore(); + final long srcPoolId = srcStore.getId(); + final ScaleIOGatewayClient client = getScaleIOClient(srcPoolId); - snapshotStore.setDataStoreId(destPoolId); - // Snapshot Id in the PowerFlex/ScaleIO pool remains the same after the migration - // Update PowerFlex snapshot name only after it is renamed, to maintain the consistency - if (renamed) { - snapshotStore.setInstallPath(ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, newSnapshotName)); - } - snapshotDataStoreDao.update(snapshotStore.getId(), snapshotStore); - } + DataStore destStore = destData.getDataStore(); + final long destPoolId = destStore.getId(); + final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); + + List snapshots = snapshotDao.listByVolumeId(srcVolumeId); + if (CollectionUtils.isNotEmpty(snapshots)) { + for (SnapshotVO snapshot : snapshots) { + SnapshotDataStoreVO snapshotStore = snapshotDataStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Primary); + if (snapshotStore == null) { + continue; } - answer = new Answer(null, true, null); - } else { - String errorMsg = "Failed to migrate PowerFlex volume: " + srcData.getId() + " to storage pool " + destPoolId; - LOGGER.debug(errorMsg); - answer = new Answer(null, false, errorMsg); + String snapshotVolumeId = ScaleIOUtil.getVolumePath(snapshotStore.getInstallPath()); + String newSnapshotName = String.format("%s-%s-%s-%s", ScaleIOUtil.SNAPSHOT_PREFIX, snapshot.getId(), + destStoragePool.getUuid().split("-")[0].substring(4), ManagementServerImpl.customCsIdentifier.value()); + boolean renamed = client.renameVolume(snapshotVolumeId, newSnapshotName); + + snapshotStore.setDataStoreId(destPoolId); + // Snapshot Id in the PowerFlex/ScaleIO pool remains the same after the migration + // Update PowerFlex snapshot name only after it is renamed, to maintain the consistency + if (renamed) { + snapshotStore.setInstallPath(ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, newSnapshotName)); + } + snapshotDataStoreDao.update(snapshotStore.getId(), snapshotStore); } + } + } + + private void revertCopyVolumeOperations(DataObject srcData, DataObject destData, Host host, String destVolumePath) { + final String srcVolumePath = ((VolumeInfo) srcData).getPath(); + final String srcVolumeFolder = ((VolumeInfo) srcData).getFolder(); + DataStore destStore = destData.getDataStore(); + + revokeAccess(destData, host, destData.getDataStore()); + String errMsg; + try { + String scaleIOVolumeId = ScaleIOUtil.getVolumePath(destVolumePath); + final ScaleIOGatewayClient client = getScaleIOClient(destStore.getId()); + Boolean deleteResult = client.deleteVolume(scaleIOVolumeId); + if (!deleteResult) { + errMsg = "Failed to delete PowerFlex volume with id: " + scaleIOVolumeId; + LOGGER.warn(errMsg); + } + } catch (Exception e) { - LOGGER.error("Failed to migrate PowerFlex volume: " + srcData.getId() + " due to: " + e.getMessage()); - answer = new Answer(null, false, e.getMessage()); + errMsg = "Unable to delete destination PowerFlex volume: " + destVolumePath + " due to " + e.getMessage(); + LOGGER.warn(errMsg); + throw new CloudRuntimeException(errMsg, e); } - return answer; + final long srcVolumeId = srcData.getId(); + if (srcVolumeId == destData.getId()) { + VolumeVO volume = volumeDao.findById(srcVolumeId); + volume.set_iScsiName(srcVolumePath); + volume.setPath(srcVolumePath); + volume.setFolder(srcVolumeFolder); + volume.setPoolId(((VolumeInfo) srcData).getPoolId()); + volumeDao.update(srcVolumeId, volume); + } } private Map getVolumeDetails(VolumeInfo volumeInfo, DataStore dataStore) { @@ -938,8 +996,6 @@ private Answer migrateVolume(DataObject srcData, DataObject destData) { final StoragePoolVO destStoragePool = storagePoolDao.findById(destPoolId); final String destStoragePoolId = destStoragePool.getPath(); int migrationTimeout = StorageManager.KvmStorageOfflineMigrationWait.value(); - LOGGER.info("HARI source volume " + srcVolumeId); - LOGGER.info("HARI destination volume " + destStoragePoolId); boolean migrateStatus = client.migrateVolume(srcVolumeId, destStoragePoolId, migrationTimeout); if (migrateStatus) { String newVolumeName = String.format("%s-%s-%s-%s", ScaleIOUtil.VOLUME_PREFIX, destData.getId(), From f482d23a0ee18018e16ae17e5fe511fff14ec0f1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 13 Mar 2023 12:51:18 +0530 Subject: [PATCH 08/31] Checking block copy status --- .../kvm/resource/LibvirtConnection.java | 2 + .../LibvirtMigrateVolumeCommandWrapper.java | 76 ++++++++++--------- .../driver/ScaleIOPrimaryDataStoreDriver.java | 47 ++++++------ 3 files changed, 67 insertions(+), 58 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index c70a72f399c7..0f8031e3aaa1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -21,6 +21,7 @@ import org.apache.log4j.Logger; import org.libvirt.Connect; +import org.libvirt.Library; import org.libvirt.LibvirtException; import com.cloud.hypervisor.Hypervisor; @@ -44,6 +45,7 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio if (conn == null) { s_logger.info("No existing libvirtd connection found. Opening a new one"); conn = new Connect(hypervisorURI, false); + Library.initEventLoop(); s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI); s_connections.put(hypervisorURI, conn); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index c78ea453250b..a083061d227c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -46,13 +46,11 @@ import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.Domain; +import org.libvirt.DomainBlockJobInfo; import org.libvirt.DomainInfo; import org.libvirt.TypedParameter; import org.libvirt.TypedUlongParameter; import org.libvirt.LibvirtException; -import org.libvirt.event.BlockJobListener; -import org.libvirt.event.BlockJobStatus; -import org.libvirt.event.BlockJobType; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { @@ -80,15 +78,6 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co String srcPath = srcVolumeObjectTO.getPath(); final String srcVolumeId = ScaleIOUtil.getVolumePath(srcVolumeObjectTO.getPath()); final String vmName = srcVolumeObjectTO.getVmName(); - Map srcDetails = command.getSrcDetails(); - final String srcSystemId = srcDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - - final String diskFileName = ScaleIOUtil.DISK_NAME_PREFIX + srcSystemId + "-" + srcVolumeId; - final String srcFilePath = ScaleIOUtil.DISK_PATH + File.separator + diskFileName; - - LOGGER.info("HARI Source volume ID: "+ srcVolumeId); - LOGGER.info("HARI source volume PATH: "+ srcFilePath); - LOGGER.info("HARI source system ID: "+ srcSystemId); // Destination Details VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); @@ -96,13 +85,11 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co final String destVolumeId = ScaleIOUtil.getVolumePath(destVolumeObjectTO.getPath()); Map destDetails = command.getDestDetails(); final String destSystemId = destDetails.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + String destDiskLabel = null; final String destDiskFileName = ScaleIOUtil.DISK_NAME_PREFIX + destSystemId + "-" + destVolumeId; final String diskFilePath = ScaleIOUtil.DISK_PATH + File.separator + destDiskFileName; - LOGGER.info("HARI destination volume ID: "+ destVolumeId); - LOGGER.info("HARI destination system ID: "+ destSystemId); - Domain dm = null; try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); @@ -123,6 +110,7 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); LibvirtVMDef.DiskDef diskdef = generateDestinationDiskDefinition(dm, srcVolumeId, srcPath, diskFilePath); + destDiskLabel = diskdef.getDiskLabel(); TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); TypedParameter[] parameters = new TypedParameter[1]; @@ -130,28 +118,41 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co Domain finalDm = dm; final Boolean[] copyStatus = {true}; - dm.blockCopy(diskdef.getDiskLabel(), diskdef.toString(), parameters, Domain.BlockCopyFlags.REUSE_EXT); - BlockJobListener listener = new BlockJobListener() { - @Override - public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { - if (type == BlockJobType.COPY && status == BlockJobStatus.READY) { - try { - finalDm.blockJobAbort(diskFilePath, Domain.BlockJobAbortFlags.PIVOT); - copyStatus[0] = false; - } catch (LibvirtException e) { - throw new RuntimeException(e); - } + dm.blockCopy(destDiskLabel, diskdef.toString(), parameters, Domain.BlockCopyFlags.REUSE_EXT); + LOGGER.info(String.format("Block copy has started for the volume %s : %s ", diskdef.getDiskLabel(), srcPath)); + + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found + int waitTimeInSec = command.getWait(); + while (waitTimeInSec > 0) { + DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(destDiskLabel, 0); + if (blockJobInfo != null) { + LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% ", destDiskLabel, srcPath, 100 * (blockJobInfo.cur / blockJobInfo.end))); + if (blockJobInfo.cur == blockJobInfo.end) { + LOGGER.debug(String.format("Block copy completed for the volume %s : %s", destDiskLabel, srcPath)); + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.PIVOT); + break; } + } else { + LOGGER.debug("Failed to get the block copy status, trying to abort the job"); + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + } + waitTimeInSec--; + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ex) { + // don't do anything } - }; - dm.addBlockJobListener(listener); - while (copyStatus[0]) { - LOGGER.info("Waiting for the block copy to complete"); } - if (copyStatus[0]) { - String msg = "Migrate volume failed due to timeout"; - LOGGER.warn(msg); + if (waitTimeInSec <= 0) { + String msg = "Block copy is taking long time, failing the job"; + LOGGER.error(msg); + try { + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } return new MigrateVolumeAnswer(command, false, msg, null); } @@ -159,6 +160,13 @@ public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobS } catch (Exception e) { String msg = "Migrate volume failed due to " + e.toString(); LOGGER.warn(msg, e); + if (destDiskLabel != null) { + try { + dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + } return new MigrateVolumeAnswer(command, false, msg, null); } finally { if (dm != null) { @@ -180,7 +188,6 @@ private LibvirtVMDef.DiskDef generateDestinationDiskDefinition(Domain dm, String LibvirtVMDef.DiskDef diskdef = null; for (final LibvirtVMDef.DiskDef disk : disks) { final String file = disk.getDiskPath(); - LOGGER.info("HARIIII disk: " + file); if (file != null && file.contains(srcVolumeId)) { diskdef = disk; break; @@ -190,7 +197,6 @@ private LibvirtVMDef.DiskDef generateDestinationDiskDefinition(Domain dm, String throw new InternalErrorException("disk: " + srcPath + " is not attached before"); } diskdef.setDiskPath(diskFilePath); - LOGGER.info("HARIIII Destination xml : " + diskdef.toString()); return diskdef; } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index b70ae125f805..9af900f5ee54 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -793,7 +793,7 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost if (migrateStatus) { updateVolumeAfterCopyVolume(srcData, destData); updateSnapshotsAfterCopyVolume(srcData, destData); - + deleteSourceVolumeAfterSuccessfulBlockCopy(srcData, host); LOGGER.debug(String.format("Successfully migrated migrate PowerFlex volume %d to storage pool %d", srcVolumeId, destPoolId)); answer = new Answer(null, true, null); } else { @@ -807,32 +807,12 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost } if (destVolumePath != null && !answer.getResult()) { - revertCopyVolumeOperations(srcData, destData, host, destVolumePath); + revertBlockCopyVolumeOperations(srcData, destData, host, destVolumePath); } return answer; } - private void checkForDestinationVolumeExistence(DataStore destStore, String destVolumePath) throws Exception { - int retryCount = 3; - while (retryCount > 0) { - try { - Thread.sleep(3000); // Try after few secs - String destScaleIOVolumeId = ScaleIOUtil.getVolumePath(destVolumePath); - final ScaleIOGatewayClient destClient = getScaleIOClient(destStore.getId()); - org.apache.cloudstack.storage.datastore.api.Volume destScaleIOVolume = destClient.getVolume(destScaleIOVolumeId); - if (destScaleIOVolume != null) { - return; - } - } catch (Exception ex) { - LOGGER.error("Exception while checking for existence of the volume at " + destVolumePath + " - " + ex.getLocalizedMessage()); - throw ex; - } finally { - retryCount--; - } - } - } - private void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData) { // destination volume is already created and volume path is set in database by this time at "CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());" final long srcVolumeId = srcData.getId(); @@ -852,6 +832,7 @@ private void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData volumeDao.update(srcVolumeId, volume); } } + private Host findEndpointForVolumeOperation(DataObject srcData) { long hostId = 0; VMInstanceVO instance = vmInstanceDao.findVMByInstanceName(((VolumeInfo) srcData).getAttachedVmName()); @@ -868,6 +849,7 @@ private Host findEndpointForVolumeOperation(DataObject srcData) { return host; } + private void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception { final long srcVolumeId = srcData.getId(); DataStore srcStore = srcData.getDataStore(); @@ -902,7 +884,26 @@ private void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destD } } - private void revertCopyVolumeOperations(DataObject srcData, DataObject destData, Host host, String destVolumePath) { + private void deleteSourceVolumeAfterSuccessfulBlockCopy(DataObject srcData, Host host) { + DataStore srcStore = srcData.getDataStore(); + String srcVolumePath = srcData.getTO().getPath(); + revokeAccess(srcData, host, srcData.getDataStore()); + String errMsg; + try { + String scaleIOVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); + final ScaleIOGatewayClient client = getScaleIOClient(srcStore.getId()); + Boolean deleteResult = client.deleteVolume(scaleIOVolumeId); + if (!deleteResult) { + errMsg = "Failed to delete source PowerFlex volume with id: " + scaleIOVolumeId; + LOGGER.warn(errMsg); + } + } catch (Exception e) { + errMsg = "Unable to delete source PowerFlex volume: " + srcVolumePath + " due to " + e.getMessage(); + LOGGER.warn(errMsg);; + } + } + + private void revertBlockCopyVolumeOperations(DataObject srcData, DataObject destData, Host host, String destVolumePath) { final String srcVolumePath = ((VolumeInfo) srcData).getPath(); final String srcVolumeFolder = ((VolumeInfo) srcData).getFolder(); DataStore destStore = destData.getDataStore(); From 5fcc6f07653b8b04e457b8e881b5e564d3feebe0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Apr 2023 13:36:24 +0530 Subject: [PATCH 09/31] Formatting code --- .../LibvirtMigrateVolumeCommandWrapper.java | 125 ++++++++++++------ .../cloud/storage/VolumeApiServiceImpl.java | 54 +++++--- 2 files changed, 127 insertions(+), 52 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index a083061d227c..a31cb765b598 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -51,6 +51,9 @@ import org.libvirt.TypedParameter; import org.libvirt.TypedUlongParameter; import org.libvirt.LibvirtException; +import org.libvirt.event.BlockJobListener; +import org.libvirt.event.BlockJobStatus; +import org.libvirt.event.BlockJobType; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { @@ -116,47 +119,11 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co TypedParameter[] parameters = new TypedParameter[1]; parameters[0] = parameter; - Domain finalDm = dm; - final Boolean[] copyStatus = {true}; dm.blockCopy(destDiskLabel, diskdef.toString(), parameters, Domain.BlockCopyFlags.REUSE_EXT); LOGGER.info(String.format("Block copy has started for the volume %s : %s ", diskdef.getDiskLabel(), srcPath)); - int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found - int waitTimeInSec = command.getWait(); - while (waitTimeInSec > 0) { - DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(destDiskLabel, 0); - if (blockJobInfo != null) { - LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% ", destDiskLabel, srcPath, 100 * (blockJobInfo.cur / blockJobInfo.end))); - if (blockJobInfo.cur == blockJobInfo.end) { - LOGGER.debug(String.format("Block copy completed for the volume %s : %s", destDiskLabel, srcPath)); - dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.PIVOT); - break; - } - } else { - LOGGER.debug("Failed to get the block copy status, trying to abort the job"); - dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); - } - waitTimeInSec--; - - try { - Thread.sleep(timeBetweenTries); - } catch (Exception ex) { - // don't do anything - } - } + return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath); - if (waitTimeInSec <= 0) { - String msg = "Block copy is taking long time, failing the job"; - LOGGER.error(msg); - try { - dm.blockJobAbort(destDiskLabel, Domain.BlockJobAbortFlags.ASYNC); - } catch (LibvirtException ex) { - LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); - } - return new MigrateVolumeAnswer(command, false, msg, null); - } - - return new MigrateVolumeAnswer(command, true, null, destPath); } catch (Exception e) { String msg = "Migrate volume failed due to " + e.toString(); LOGGER.warn(msg, e); @@ -179,6 +146,90 @@ private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand co } } + private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath) throws LibvirtException { + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found + int waitTimeInSec = command.getWait(); + while (waitTimeInSec > 0) { + DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0); + if (blockJobInfo != null) { + LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% ", diskLabel, srcPath, 100 * (blockJobInfo.cur / blockJobInfo.end))); + if (blockJobInfo.cur == blockJobInfo.end) { + LOGGER.debug(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); + break; + } + } else { + LOGGER.debug("Failed to get the block copy status, trying to abort the job"); + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } + waitTimeInSec--; + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ex) { + // don't do anything + } + } + + if (waitTimeInSec <= 0) { + String msg = "Block copy is taking long time, failing the job"; + LOGGER.error(msg); + try { + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + return new MigrateVolumeAnswer(command, false, msg, null); + } + + return new MigrateVolumeAnswer(command, true, null, destPath); + } + + private MigrateVolumeAnswer checkBlockJobStatusUsingListener(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath) throws LibvirtException, InterruptedException { + final Boolean[] copyStatus = {false}; + + BlockJobListener listener = new BlockJobListener() { + @Override + public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { + LOGGER.info("waiting for the event"); + if (type == BlockJobType.COPY && status == BlockJobStatus.READY) { + try { + domain.blockJobAbort(diskPath, Domain.BlockJobAbortFlags.PIVOT); + copyStatus[0] = true; + } catch (LibvirtException e) { + throw new RuntimeException(e); + } + } + } + }; + dm.addBlockJobListener(listener); + + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found + int waitTimeInSec = command.getWait(); + while (waitTimeInSec > 0 && !copyStatus[0]) { + waitTimeInSec--; + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ex) { + // don't do anything + } + LOGGER.info("Waiting for the block copy to complete"); + } + + if (!copyStatus[0]) { + String msg = "Block copy is taking long time, failing the job"; + LOGGER.error(msg); + try { + dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); + } catch (LibvirtException ex) { + LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); + } + return new MigrateVolumeAnswer(command, false, msg, null); + } + + return new MigrateVolumeAnswer(command, true, null, destPath); + } + private LibvirtVMDef.DiskDef generateDestinationDiskDefinition(Domain dm, String srcVolumeId, String srcPath, String diskFilePath) throws InternalErrorException, LibvirtException { final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); final String domXml = dm.getXMLDesc(0); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 15bb5fca60d2..de343e3739e8 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -88,12 +88,7 @@ import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.*; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.identity.ManagementServerNode; @@ -157,12 +152,7 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.dao.DiskOfferingDao; -import com.cloud.storage.dao.SnapshotDao; -import com.cloud.storage.dao.StoragePoolTagsDao; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.storage.dao.*; import com.cloud.storage.snapshot.SnapshotApiService; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.template.TemplateManager; @@ -326,6 +316,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject protected ProjectManager projectManager; + @Inject + protected StoragePoolDetailsDao storagePoolDetailsDao; protected Gson _gson; @@ -1098,8 +1090,8 @@ public VolumeVO resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationExcep if (isNotPossibleToResize(volume, diskOffering)) { throw new InvalidParameterValueException( "Failed to resize Root volume. The service offering of this Volume has been configured with a root disk size; " - + "on such case a Root Volume can only be resized when changing to another Service Offering with a Root disk size. " - + "For more details please check out the Official Resizing Volumes documentation."); + "on such case a Root Volume can only be resized when changing to another Service Offering with a Root disk size. " + "For more details please check out the Official Resizing Volumes documentation."); } // convert from bytes to GiB @@ -1246,7 +1238,7 @@ public VolumeVO resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationExcep */ if (currentSize > newSize && !shrinkOk) { throw new InvalidParameterValueException("Going from existing size of " + currentSize + " to size of " + newSize + " would shrink the volume." - + "Need to sign off by supplying the shrinkok parameter with value of true."); + "Need to sign off by supplying the shrinkok parameter with value of true."); } if (newSize > currentSize) { @@ -3019,6 +3011,14 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { } } + // Offline volume migration check for scaleIO volumes across scaleio clusters + if (vm == null || !State.Running.equals(vm.getState())) { + StoragePoolVO sourceStoragePoolVO = _storagePoolDao.findById(vol.getPoolId()); + if (sourceStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && isScaleIOVolumeOnDifferentScaleIOStorageInstances(vol.getPoolId(), storagePoolId)) { + throw new InvalidParameterValueException("Volume needs to be attached to a VM to move across ScaleIO storages in different ScaleIO clusters"); + } + } + if (vm != null && HypervisorType.VMware.equals(vm.getHypervisorType()) && State.Stopped.equals(vm.getState())) { @@ -3152,6 +3152,30 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { return orchestrateMigrateVolume(vol, destPool, liveMigrateVolume, newDiskOffering); } + private boolean isScaleIOVolumeOnDifferentScaleIOStorageInstances(long srcPoolId, long destPoolId) { + String srcPoolSystemId = null; + StoragePoolDetailVO srcPoolSystemIdDetail = storagePoolDetailsDao.findDetail(srcPoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + if (srcPoolSystemIdDetail != null) { + srcPoolSystemId = srcPoolSystemIdDetail.getValue(); + } + + String destPoolSystemId = null; + StoragePoolDetailVO destPoolSystemIdDetail = storagePoolDetailsDao.findDetail(destPoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + if (destPoolSystemIdDetail != null) { + destPoolSystemId = destPoolSystemIdDetail.getValue(); + } + + if (StringUtils.isAnyEmpty(srcPoolSystemId, destPoolSystemId)) { + throw new CloudRuntimeException("Failed to validate PowerFlex pools compatibility for migration as storage instance details are not available"); + } + + if (srcPoolSystemId.equals(destPoolSystemId)) { + return true; + } + + return false; + } + private boolean isSourceOrDestNotOnStorPool(StoragePoolVO storagePoolVO, StoragePoolVO destinationStoragePoolVo) { return storagePoolVO.getPoolType() != Storage.StoragePoolType.StorPool || destinationStoragePoolVo.getPoolType() != Storage.StoragePoolType.StorPool; From 4ca5dfad666d94ae10a825017983bed1f7353b18 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Apr 2023 13:43:04 +0530 Subject: [PATCH 10/31] Fixed failures --- .../vm/VirtualMachineManagerImplTest.java | 16 +++++++++++ .../cloud/storage/VolumeApiServiceImpl.java | 27 ++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 742bb3dda897..fd3dba6dbd50 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -32,6 +32,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.ConfigKey; @@ -46,6 +48,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; +import org.mockito.stubbing.Answer; import org.mockito.runners.MockitoJUnitRunner; import com.cloud.agent.AgentManager; @@ -68,6 +71,7 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; @@ -376,6 +380,12 @@ public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentS @Test public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + // return any storage type except powerflex/scaleio + List values = Arrays.asList(Storage.StoragePoolType.values()); + when(storagePoolVoMock.getPoolType()).thenAnswer((Answer) invocation -> { + List filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList()); + int randomIndex = new Random().nextInt(filteredValues.size()); + return filteredValues.get(randomIndex); }); virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, storagePoolVoMock); @@ -386,6 +396,12 @@ public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentS @Test(expected = CloudRuntimeException.class) public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolNotEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + // return any storage type except powerflex/scaleio + List values = Arrays.asList(Storage.StoragePoolType.values()); + when(storagePoolVoMock.getPoolType()).thenAnswer((Answer) invocation -> { + List filteredValues = values.stream().filter(v -> v != Storage.StoragePoolType.PowerFlex).collect(Collectors.toList()); + int randomIndex = new Random().nextInt(filteredValues.size()); + return filteredValues.get(randomIndex); }); virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class)); } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index de343e3739e8..8f0525a1dc52 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -88,7 +88,14 @@ import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.datastore.db.*; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.identity.ManagementServerNode; @@ -152,7 +159,12 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.dao.*; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.StoragePoolTagsDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.storage.snapshot.SnapshotApiService; import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.template.TemplateManager; @@ -1090,8 +1102,8 @@ public VolumeVO resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationExcep if (isNotPossibleToResize(volume, diskOffering)) { throw new InvalidParameterValueException( "Failed to resize Root volume. The service offering of this Volume has been configured with a root disk size; " - "on such case a Root Volume can only be resized when changing to another Service Offering with a Root disk size. " - "For more details please check out the Official Resizing Volumes documentation."); + + "on such case a Root Volume can only be resized when changing to another Service Offering with a Root disk size. " + + "For more details please check out the Official Resizing Volumes documentation."); } // convert from bytes to GiB @@ -1238,7 +1250,7 @@ public VolumeVO resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationExcep */ if (currentSize > newSize && !shrinkOk) { throw new InvalidParameterValueException("Going from existing size of " + currentSize + " to size of " + newSize + " would shrink the volume." - "Need to sign off by supplying the shrinkok parameter with value of true."); + + "Need to sign off by supplying the shrinkok parameter with value of true."); } if (newSize > currentSize) { @@ -3153,14 +3165,15 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { } private boolean isScaleIOVolumeOnDifferentScaleIOStorageInstances(long srcPoolId, long destPoolId) { + String SCALEIO_STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id"; String srcPoolSystemId = null; - StoragePoolDetailVO srcPoolSystemIdDetail = storagePoolDetailsDao.findDetail(srcPoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + StoragePoolDetailVO srcPoolSystemIdDetail = storagePoolDetailsDao.findDetail(srcPoolId, SCALEIO_STORAGE_POOL_SYSTEM_ID); if (srcPoolSystemIdDetail != null) { srcPoolSystemId = srcPoolSystemIdDetail.getValue(); } String destPoolSystemId = null; - StoragePoolDetailVO destPoolSystemIdDetail = storagePoolDetailsDao.findDetail(destPoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); + StoragePoolDetailVO destPoolSystemIdDetail = storagePoolDetailsDao.findDetail(destPoolId, SCALEIO_STORAGE_POOL_SYSTEM_ID); if (destPoolSystemIdDetail != null) { destPoolSystemId = destPoolSystemIdDetail.getValue(); } From 7c7348a84dc4a2944290f5d9cce13b452861b382 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 28 Mar 2023 08:07:15 +0530 Subject: [PATCH 11/31] code refactoring and some changes --- .../wrapper/LibvirtMigrateVolumeCommandWrapper.java | 6 +++--- .../datastore/driver/ScaleIOPrimaryDataStoreDriver.java | 2 +- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index a31cb765b598..bfc77ddbb296 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -32,13 +32,13 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; import java.io.File; import java.util.List; import java.util.Map; import java.util.UUID; -import com.cloud.storage.Storage; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -66,7 +66,7 @@ public Answer execute(final MigrateVolumeCommand command, final LibvirtComputing MigrateVolumeAnswer answer; if (srcPrimaryDataStore.getPoolType().equals(Storage.StoragePoolType.PowerFlex)) { - answer = migrateVolumeInternal(command, libvirtComputingResource); + answer = migratePowerFlexVolume(command, libvirtComputingResource); } else { answer = migrateRegularVolume(command, libvirtComputingResource); } @@ -74,7 +74,7 @@ public Answer execute(final MigrateVolumeCommand command, final LibvirtComputing return answer; } - private MigrateVolumeAnswer migrateVolumeInternal (final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { // Source Details VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 9af900f5ee54..5d6d6f372ac9 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -762,7 +762,7 @@ private Answer copyTemplateToVolume(DataObject srcData, DataObject destData, Hos } private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost) { - // Volume migration within same PowerFlex/ScaleIO cluster (with same System ID) + // Volume migration across different PowerFlex/ScaleIO clusters final long srcVolumeId = srcData.getId(); DataStore srcStore = srcData.getDataStore(); Map srcDetails = getVolumeDetails((VolumeInfo) srcData, srcStore); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 8f0525a1dc52..01a29e38747e 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -3027,7 +3027,7 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { if (vm == null || !State.Running.equals(vm.getState())) { StoragePoolVO sourceStoragePoolVO = _storagePoolDao.findById(vol.getPoolId()); if (sourceStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && isScaleIOVolumeOnDifferentScaleIOStorageInstances(vol.getPoolId(), storagePoolId)) { - throw new InvalidParameterValueException("Volume needs to be attached to a VM to move across ScaleIO storages in different ScaleIO clusters"); + throw new InvalidParameterValueException("Volume needs to be attached to a running VM to move across ScaleIO storages in different ScaleIO clusters"); } } @@ -3182,7 +3182,7 @@ private boolean isScaleIOVolumeOnDifferentScaleIOStorageInstances(long srcPoolId throw new CloudRuntimeException("Failed to validate PowerFlex pools compatibility for migration as storage instance details are not available"); } - if (srcPoolSystemId.equals(destPoolSystemId)) { + if (!srcPoolSystemId.equals(destPoolSystemId)) { return true; } From 8aa72248eabbb35afcc9614834317c1007fd534d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 30 Mar 2023 14:08:32 +0530 Subject: [PATCH 12/31] Removed unused methods --- .../LibvirtMigrateVolumeCommandWrapper.java | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index bfc77ddbb296..d808405410b3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -185,51 +185,6 @@ private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Do return new MigrateVolumeAnswer(command, true, null, destPath); } - private MigrateVolumeAnswer checkBlockJobStatusUsingListener(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath) throws LibvirtException, InterruptedException { - final Boolean[] copyStatus = {false}; - - BlockJobListener listener = new BlockJobListener() { - @Override - public void onEvent(Domain domain, String diskPath, BlockJobType type, BlockJobStatus status) { - LOGGER.info("waiting for the event"); - if (type == BlockJobType.COPY && status == BlockJobStatus.READY) { - try { - domain.blockJobAbort(diskPath, Domain.BlockJobAbortFlags.PIVOT); - copyStatus[0] = true; - } catch (LibvirtException e) { - throw new RuntimeException(e); - } - } - } - }; - dm.addBlockJobListener(listener); - - int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found - int waitTimeInSec = command.getWait(); - while (waitTimeInSec > 0 && !copyStatus[0]) { - waitTimeInSec--; - try { - Thread.sleep(timeBetweenTries); - } catch (Exception ex) { - // don't do anything - } - LOGGER.info("Waiting for the block copy to complete"); - } - - if (!copyStatus[0]) { - String msg = "Block copy is taking long time, failing the job"; - LOGGER.error(msg); - try { - dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); - } catch (LibvirtException ex) { - LOGGER.error("Migrate volume failed while aborting the block job due to " + ex.getMessage()); - } - return new MigrateVolumeAnswer(command, false, msg, null); - } - - return new MigrateVolumeAnswer(command, true, null, destPath); - } - private LibvirtVMDef.DiskDef generateDestinationDiskDefinition(Domain dm, String srcVolumeId, String srcPath, String diskFilePath) throws InternalErrorException, LibvirtException { final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); final String domXml = dm.getXMLDesc(0); From a148bfa7b3dfb6999fbab9c489187f6987323fa4 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 4 Apr 2023 15:56:57 +0530 Subject: [PATCH 13/31] removed unused imports --- .../resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index d808405410b3..2e7a4192c569 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -51,9 +51,6 @@ import org.libvirt.TypedParameter; import org.libvirt.TypedUlongParameter; import org.libvirt.LibvirtException; -import org.libvirt.event.BlockJobListener; -import org.libvirt.event.BlockJobStatus; -import org.libvirt.event.BlockJobType; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { From dee41d9627ce0fd453af493c943df9df558dbbcd Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Apr 2023 13:52:31 +0530 Subject: [PATCH 14/31] Unit tests to check if volume belongs to same or different storage scaleio cluster --- .../driver/ScaleIOPrimaryDataStoreDriver.java | 2 +- .../ScaleIOPrimaryDataStoreDriverTest.java | 114 ++++++++++++++++++ .../cloud/storage/VolumeApiServiceImpl.java | 2 +- .../storage/VolumeApiServiceImplTest.java | 57 +++++++++ 4 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 5d6d6f372ac9..c16cdaa5f41b 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -1068,7 +1068,7 @@ private Answer migrateVolume(DataObject srcData, DataObject destData) { return answer; } - private boolean isSameScaleIOStorageInstance(DataStore srcStore, DataStore destStore) { + public boolean isSameScaleIOStorageInstance(DataStore srcStore, DataStore destStore) { long srcPoolId = srcStore.getId(); String srcPoolSystemId = null; StoragePoolDetailVO srcPoolSystemIdDetail = storagePoolDetailsDao.findDetail(srcPoolId, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java new file mode 100644 index 000000000000..4ed31e6d8c14 --- /dev/null +++ b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java @@ -0,0 +1,114 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.storage.datastore.driver.client; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.driver.ScaleIOPrimaryDataStoreDriver; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ScaleIOPrimaryDataStoreDriverTest { + + @Spy + @InjectMocks + ScaleIOPrimaryDataStoreDriver scaleIOPrimaryDataStoreDriver; + + @Mock + StoragePoolDetailsDao storagePoolDetailsDao; + + @Test + public void testSameScaleIOStorageInstance() { + DataStore srcStore = Mockito.mock(DataStore.class); + DataStore destStore = Mockito.mock(DataStore.class); + when(srcStore.getId()).thenReturn(1L); + when(destStore.getId()).thenReturn(2L); + + StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String srcPoolSystemId = "610204d03e3ad60f"; + when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); + + StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String destPoolSystemId = "610204d03e3ad60f"; + when(destPoolSystemIdDetail.getValue()).thenReturn(destPoolSystemId); + + when(storagePoolDetailsDao.findDetail(1L,ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(srcPoolSystemIdDetail); + when(storagePoolDetailsDao.findDetail(2L,ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(destPoolSystemIdDetail); + + boolean result = scaleIOPrimaryDataStoreDriver.isSameScaleIOStorageInstance(srcStore, destStore); + + Assert.assertTrue(result); + } + + @Test + public void testDifferentScaleIOStorageInstance() { + DataStore srcStore = Mockito.mock(DataStore.class); + DataStore destStore = Mockito.mock(DataStore.class); + when(srcStore.getId()).thenReturn(1L); + when(destStore.getId()).thenReturn(2L); + + StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String srcPoolSystemId = "610204d03e3ad60f"; + when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); + + StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String destPoolSystemId = "7332760565f6340f"; + when(destPoolSystemIdDetail.getValue()).thenReturn(destPoolSystemId); + + when(storagePoolDetailsDao.findDetail(1L,ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(srcPoolSystemIdDetail); + when(storagePoolDetailsDao.findDetail(2L,ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(destPoolSystemIdDetail); + + boolean result = scaleIOPrimaryDataStoreDriver.isSameScaleIOStorageInstance(srcStore, destStore); + + Assert.assertFalse(result); + } + + @Test (expected = CloudRuntimeException.class) + public void testCheckVolumeOnDifferentScaleIOStorageInstanceSystemIdShouldNotBeNull() { + DataStore srcStore = Mockito.mock(DataStore.class); + DataStore destStore = Mockito.mock(DataStore.class); + when(srcStore.getId()).thenReturn(1L); + when(destStore.getId()).thenReturn(2L); + + StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String srcPoolSystemId = "610204d03e3ad60f"; + when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); + + StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + when(destPoolSystemIdDetail.getValue()).thenReturn(null); + + when(storagePoolDetailsDao.findDetail(1L,ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(srcPoolSystemIdDetail); + when(storagePoolDetailsDao.findDetail(2L,ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(destPoolSystemIdDetail); + + scaleIOPrimaryDataStoreDriver.isSameScaleIOStorageInstance(srcStore, destStore); + } +} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 01a29e38747e..7c2e1db6f23c 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -3164,7 +3164,7 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { return orchestrateMigrateVolume(vol, destPool, liveMigrateVolume, newDiskOffering); } - private boolean isScaleIOVolumeOnDifferentScaleIOStorageInstances(long srcPoolId, long destPoolId) { + protected boolean isScaleIOVolumeOnDifferentScaleIOStorageInstances(long srcPoolId, long destPoolId) { String SCALEIO_STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id"; String srcPoolSystemId = null; StoragePoolDetailVO srcPoolSystemIdDetail = storagePoolDetailsDao.findDetail(srcPoolId, SCALEIO_STORAGE_POOL_SYSTEM_ID); diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 16c7887ef915..a0ece775c928 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -54,6 +54,8 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; @@ -113,6 +115,7 @@ import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; @@ -210,6 +213,8 @@ public class VolumeApiServiceImplTest { @Mock private ProjectManager projectManagerMock; + @Mock + private StoragePoolDetailsDao storagePoolDetailsDao; private long accountMockId = 456l; private long volumeMockId = 12313l; private long vmInstanceMockId = 1123l; @@ -1563,4 +1568,56 @@ public void testIsSendCommandForVmVolumeAttachDetachKVMHost() { Mockito.when(host.getHypervisorType()).thenReturn(HypervisorType.KVM); Assert.assertFalse(volumeApiServiceImpl.isSendCommandForVmVolumeAttachDetach(host, Mockito.mock(StoragePoolVO.class))); } + + @Test + public void testVolumeOnDifferentScaleIOStorageInstance() { + StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String srcPoolSystemId = "610204d03e3ad60f"; + when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); + + StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String destPoolSystemId = "7332760565f6340f"; + when(destPoolSystemIdDetail.getValue()).thenReturn(destPoolSystemId); + + when(storagePoolDetailsDao.findDetail(1L,"powerflex.storagepool.system.id")).thenReturn(srcPoolSystemIdDetail); + when(storagePoolDetailsDao.findDetail(2L,"powerflex.storagepool.system.id")).thenReturn(destPoolSystemIdDetail); + + boolean result = volumeApiServiceImpl.isScaleIOVolumeOnDifferentScaleIOStorageInstances(1L, 2L); + + Assert.assertTrue(result); + } + + @Test + public void testVolumeOnSameScaleIOStorageInstance() { + StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String srcPoolSystemId = "610204d03e3ad60f"; + when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); + + StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String destPoolSystemId = "610204d03e3ad60f"; + when(destPoolSystemIdDetail.getValue()).thenReturn(destPoolSystemId); + + when(storagePoolDetailsDao.findDetail(1L,"powerflex.storagepool.system.id")).thenReturn(srcPoolSystemIdDetail); + when(storagePoolDetailsDao.findDetail(2L,"powerflex.storagepool.system.id")).thenReturn(destPoolSystemIdDetail); + + boolean result = volumeApiServiceImpl.isScaleIOVolumeOnDifferentScaleIOStorageInstances(1L, 2L); + + Assert.assertFalse(result); + } + + @Test (expected = CloudRuntimeException.class) + public void testCheckVolumeOnDifferentScaleIOStorageInstanceSystemIdShouldNotBeNull() { + StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + String srcPoolSystemId = "610204d03e3ad60f"; + when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); + + StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); + when(destPoolSystemIdDetail.getValue()).thenReturn(null); + + when(storagePoolDetailsDao.findDetail(1L,"powerflex.storagepool.system.id")).thenReturn(srcPoolSystemIdDetail); + when(storagePoolDetailsDao.findDetail(2L,"powerflex.storagepool.system.id")).thenReturn(destPoolSystemIdDetail); + + volumeApiServiceImpl.isScaleIOVolumeOnDifferentScaleIOStorageInstances(1L, 2L); + } + } From 28018a302ee4373d235d5bdcaeae46fb462b145d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Apr 2023 13:54:45 +0530 Subject: [PATCH 15/31] Unit tests for volume livemigration in ScaleIOPrimaryDataStoreDriver --- .../driver/ScaleIOPrimaryDataStoreDriver.java | 16 +- .../ScaleIOPrimaryDataStoreDriverTest.java | 351 +++++++++++++++++- 2 files changed, 355 insertions(+), 12 deletions(-) diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index c16cdaa5f41b..3d846de084d8 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -136,7 +136,7 @@ public ScaleIOPrimaryDataStoreDriver() { } - private ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId) throws Exception { + public ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId) throws Exception { return ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePoolId, storagePoolDetailsDao); } @@ -448,7 +448,7 @@ public void revertSnapshot(SnapshotInfo snapshot, SnapshotInfo snapshotOnPrimary } } - private CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId) { + public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId) { LOGGER.debug("Creating PowerFlex volume"); StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); @@ -687,7 +687,7 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As if (isSameScaleIOStorageInstance(srcStore, destStore)) { answer = migrateVolume(srcData, destData); } else { - answer = copyVolume(srcData, destData, destHost); + answer = copyVolume(srcData, destData); } } else { errMsg = "Unsupported copy operation from src object: (" + srcData.getType() + ", " + srcData.getDataStore() + "), dest object: (" @@ -761,7 +761,7 @@ private Answer copyTemplateToVolume(DataObject srcData, DataObject destData, Hos return answer; } - private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost) { + public Answer copyVolume(DataObject srcData, DataObject destData) { // Volume migration across different PowerFlex/ScaleIO clusters final long srcVolumeId = srcData.getId(); DataStore srcStore = srcData.getDataStore(); @@ -813,7 +813,7 @@ private Answer copyVolume(DataObject srcData, DataObject destData, Host destHost return answer; } - private void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData) { + public void updateVolumeAfterCopyVolume(DataObject srcData, DataObject destData) { // destination volume is already created and volume path is set in database by this time at "CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId());" final long srcVolumeId = srcData.getId(); final long destVolumeId = destData.getId(); @@ -850,7 +850,7 @@ private Host findEndpointForVolumeOperation(DataObject srcData) { return host; } - private void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception { + public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception { final long srcVolumeId = srcData.getId(); DataStore srcStore = srcData.getDataStore(); final long srcPoolId = srcStore.getId(); @@ -884,7 +884,7 @@ private void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destD } } - private void deleteSourceVolumeAfterSuccessfulBlockCopy(DataObject srcData, Host host) { + public void deleteSourceVolumeAfterSuccessfulBlockCopy(DataObject srcData, Host host) { DataStore srcStore = srcData.getDataStore(); String srcVolumePath = srcData.getTO().getPath(); revokeAccess(srcData, host, srcData.getDataStore()); @@ -903,7 +903,7 @@ private void deleteSourceVolumeAfterSuccessfulBlockCopy(DataObject srcData, Host } } - private void revertBlockCopyVolumeOperations(DataObject srcData, DataObject destData, Host host, String destVolumePath) { + public void revertBlockCopyVolumeOperations(DataObject srcData, DataObject destData, Host host, String destVolumePath) { final String srcVolumePath = ((VolumeInfo) srcData).getPath(); final String srcVolumeFolder = ((VolumeInfo) srcData).getFolder(); DataStore destStore = destData.getDataStore(); diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java index 4ed31e6d8c14..643bf9165b7c 100644 --- a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java +++ b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java @@ -19,33 +19,84 @@ package org.apache.cloudstack.storage.datastore.driver.client; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.MigrateVolumeAnswer; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.configuration.Config; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.storage.Storage; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.RemoteHostEndPoint; +import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.driver.ScaleIOPrimaryDataStoreDriver; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(RemoteHostEndPoint.class) public class ScaleIOPrimaryDataStoreDriverTest { @Spy @InjectMocks - ScaleIOPrimaryDataStoreDriver scaleIOPrimaryDataStoreDriver; - + ScaleIOPrimaryDataStoreDriver scaleIOPrimaryDataStoreDriver = new ScaleIOPrimaryDataStoreDriver(); @Mock StoragePoolDetailsDao storagePoolDetailsDao; + @Mock + PrimaryDataStoreDao storagePoolDao; + @Mock + VolumeDao volumeDao; + @Mock + VolumeDetailsDao volumeDetailsDao; + @Mock + VolumeService volumeService; + @Mock + VMInstanceDao vmInstanceDao; + @Mock + HostDao hostDao; + @Mock + ConfigurationDao configDao; + @Before + public void initMocks() { + MockitoAnnotations.initMocks(this); + } @Test public void testSameScaleIOStorageInstance() { DataStore srcStore = Mockito.mock(DataStore.class); @@ -111,4 +162,296 @@ public void testCheckVolumeOnDifferentScaleIOStorageInstanceSystemIdShouldNotBeN scaleIOPrimaryDataStoreDriver.isSameScaleIOStorageInstance(srcStore, destStore); } + + @Test + public void testMigrateVolumeWithinSameScaleIOClusterSuccess() throws Exception { + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + VolumeInfo destData = Mockito.mock(VolumeInfo.class); + + DataStore srcStore = Mockito.mock(DataStore.class); + DataStore destStore = Mockito.mock(DataStore.class); + + when(srcData.getDataStore()).thenReturn(srcStore); + when(destData.getDataStore()).thenReturn(destStore); + + fillSrcVolumeDetails(srcData, srcStore); + fillDestVolumeDetails(destData, destStore); + + VolumeObjectTO destVolTO = Mockito.mock(VolumeObjectTO.class); + when(destData.getTO()).thenReturn(destVolTO); + Host host = prepareEndpointForVolumeOperation(srcData); + PowerMockito.mockStatic(RemoteHostEndPoint.class); + RemoteHostEndPoint ep = Mockito.mock(RemoteHostEndPoint.class); + when(RemoteHostEndPoint.getHypervisorHostEndPoint(host)).thenReturn(ep); + + DataTO dataTO = Mockito.mock(DataTO.class); + CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO); + doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L); + when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee"); + doReturn(true).when(scaleIOPrimaryDataStoreDriver) + .grantAccess(any(), any(), any()); + + when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600"); + MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class); + when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer); + when(migrateVolumeAnswer.getResult()).thenReturn(true); + + Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver) + .updateVolumeAfterCopyVolume(any(), any()); + Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver) + .updateSnapshotsAfterCopyVolume(any(), any()); + Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver) + .deleteSourceVolumeAfterSuccessfulBlockCopy(any(), any()); + + Answer answer = scaleIOPrimaryDataStoreDriver.copyVolume(srcData, destData); + + Assert.assertTrue(answer.getResult()); + } + + @Test + public void testMigrateVolumeWithinSameScaleIOClusterFailure() throws Exception { + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + VolumeInfo destData = Mockito.mock(VolumeInfo.class); + + DataStore srcStore = Mockito.mock(DataStore.class); + DataStore destStore = Mockito.mock(DataStore.class); + + when(srcData.getDataStore()).thenReturn(srcStore); + when(destData.getDataStore()).thenReturn(destStore); + + fillSrcVolumeDetails(srcData, srcStore); + fillDestVolumeDetails(destData, destStore); + + VolumeObjectTO destVolTO = Mockito.mock(VolumeObjectTO.class); + when(destData.getTO()).thenReturn(destVolTO); + Host host = prepareEndpointForVolumeOperation(srcData); + PowerMockito.mockStatic(RemoteHostEndPoint.class); + RemoteHostEndPoint ep = Mockito.mock(RemoteHostEndPoint.class); + when(RemoteHostEndPoint.getHypervisorHostEndPoint(host)).thenReturn(ep); + + DataTO dataTO = Mockito.mock(DataTO.class); + CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO); + doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L); + when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee"); + doReturn(true).when(scaleIOPrimaryDataStoreDriver) + .grantAccess(any(), any(), any()); + + when(configDao.getValue(Config.MigrateWait.key())).thenReturn("3600"); + MigrateVolumeAnswer migrateVolumeAnswer = Mockito.mock(MigrateVolumeAnswer.class); + when(ep.sendMessage(any())).thenReturn(migrateVolumeAnswer); + when(migrateVolumeAnswer.getResult()).thenReturn(false); + Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver) + .revertBlockCopyVolumeOperations(any(), any(), any(), any()); + + Answer answer = scaleIOPrimaryDataStoreDriver.copyVolume(srcData, destData); + + Assert.assertFalse(answer.getResult()); + } + + private void fillSrcVolumeDetails(VolumeInfo srcData, DataStore srcStore) { + when(srcStore.getId()).thenReturn(1L); + when(srcData.getId()).thenReturn(1L); + + StoragePoolVO storagePoolVO = Mockito.mock(StoragePoolVO.class); + when(storagePoolDao.findById(1L)).thenReturn(storagePoolVO); + when(storagePoolVO.isManaged()).thenReturn(true); + + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + when(volumeDao.findById(1L)).thenReturn(volumeVO); + + when(volumeDetailsDao.findDetail(1L, DiskTO.SCSI_NAA_DEVICE_ID)).thenReturn(null); + when(volumeService.getChapInfo(srcData, srcStore)).thenReturn(null); + + StoragePoolDetailVO srcStoragePoolDetail = Mockito.mock(StoragePoolDetailVO.class); + when(srcStoragePoolDetail.getValue()).thenReturn("610204d03e3ad60f"); + when(storagePoolDetailsDao.findDetail(1L, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(srcStoragePoolDetail); + } + + private void fillDestVolumeDetails(VolumeInfo srcData, DataStore srcStore) { + when(srcStore.getId()).thenReturn(2L); + when(srcData.getId()).thenReturn(2L); + + StoragePoolVO storagePoolVO = Mockito.mock(StoragePoolVO.class); + when(storagePoolDao.findById(2L)).thenReturn(storagePoolVO); + when(storagePoolVO.isManaged()).thenReturn(true); + + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + when(volumeDao.findById(2L)).thenReturn(volumeVO); + + when(volumeDetailsDao.findDetail(2L, DiskTO.SCSI_NAA_DEVICE_ID)).thenReturn(null); + when(volumeService.getChapInfo(srcData, srcStore)).thenReturn(null); + + StoragePoolDetailVO srcStoragePoolDetail = Mockito.mock(StoragePoolDetailVO.class); + when(srcStoragePoolDetail.getValue()).thenReturn("7332760565f6340f"); + when(storagePoolDetailsDao.findDetail(2L, ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)).thenReturn(srcStoragePoolDetail); + } + + private Host prepareEndpointForVolumeOperation(VolumeInfo srcData) { + VMInstanceVO instance = Mockito.mock(VMInstanceVO.class); + when(srcData.getAttachedVmName()).thenReturn("i-2-VM"); + when(vmInstanceDao.findVMByInstanceName("i-2-VM")).thenReturn(instance); + when(instance.getHostId()).thenReturn(4L); + when(instance.getState()).thenReturn(VirtualMachine.State.Running); + HostVO host = Mockito.mock(HostVO.class); + when(hostDao.findById(4L)).thenReturn(host); + + return host; + } + + @Test + public void updateVolumeAfterCopyVolumeLiveMigrate() { + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + VolumeInfo destData = Mockito.mock(VolumeInfo.class); + + when(srcData.getId()).thenReturn(1L); + when(destData.getId()).thenReturn(1L); + + VolumeVO volume = new VolumeVO("root", 1L, 1L, 1L, 1L, 1L, "root", "root", Storage.ProvisioningType.THIN, 1, null, null, "root", Volume.Type.ROOT); + volume.setPoolId(2L); + when(volumeDao.findById(1L)).thenReturn(volume); + when(volumeDao.update(1L, volume)).thenReturn(true); + + scaleIOPrimaryDataStoreDriver.updateVolumeAfterCopyVolume(srcData, destData); + + Assert.assertEquals(Optional.of(2L), Optional.of(volume.getLastPoolId())); + } + + @Test + public void updateVolumeAfterCopyVolumeOffline() { + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + VolumeInfo destData = Mockito.mock(VolumeInfo.class); + + when(srcData.getId()).thenReturn(1L); + when(destData.getId()).thenReturn(2L); + + VolumeVO volume = new VolumeVO("root", 1L, 1L, 1L, 1L, 1L, "root", "root", Storage.ProvisioningType.THIN, 1, null, null, "root", Volume.Type.ROOT); + when(volumeDao.findById(1L)).thenReturn(volume); + when(volumeDao.update(1L, volume)).thenReturn(true); + + scaleIOPrimaryDataStoreDriver.updateVolumeAfterCopyVolume(srcData, destData); + + Assert.assertNull(volume.get_iScsiName()); + Assert.assertNull(volume.getPath()); + Assert.assertNull(volume.getFolder()); + } + + @Test + public void revertBlockCopyVolumeOperationsOnDeleteSuccess() throws Exception{ + //Either destination volume delete success or failure, DB operations should get revert + + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + VolumeInfo destData = Mockito.mock(VolumeInfo.class); + Host host = Mockito.mock(Host.class); + String destVolumePath = "01b332b300000007:vol-11-b9e2-10ee"; + + when(srcData.getId()).thenReturn(1L); + when(srcData.getPoolId()).thenReturn(1L); + when(destData.getId()).thenReturn(1L); + + when(srcData.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee"); + when(srcData.getFolder()).thenReturn("921c364500000007"); + DataStore destStore = Mockito.mock(DataStore.class); + when(destStore.getId()).thenReturn(2L); + when(destData.getDataStore()).thenReturn(destStore); + doNothing().when(scaleIOPrimaryDataStoreDriver) + .revokeAccess(any(), any(), any()); + + ScaleIOGatewayClient client = Mockito.mock(ScaleIOGatewayClient.class); + doReturn(client).when(scaleIOPrimaryDataStoreDriver) + .getScaleIOClient(any()); + when(client.deleteVolume(any())).thenReturn(true); + + VolumeVO volume = new VolumeVO("root", 1L, 1L, 1L, 1L, 1L, "root", "root", Storage.ProvisioningType.THIN, 1, null, null, "root", Volume.Type.ROOT); + when(volumeDao.findById(1L)).thenReturn(volume); + when(volumeDao.update(1L, volume)).thenReturn(true); + + scaleIOPrimaryDataStoreDriver.revertBlockCopyVolumeOperations(srcData, destData, host, destVolumePath); + + Assert.assertEquals("bec0ba7700000007:vol-11-6aef-10ee", volume.get_iScsiName()); + Assert.assertEquals("bec0ba7700000007:vol-11-6aef-10ee", volume.getPath()); + Assert.assertEquals("921c364500000007", volume.getFolder()); + } + + @Test + public void revertBlockCopyVolumeOperationsOnDeleteFailure() throws Exception{ + //Either destination volume delete success or failure, DB operations should get revert + + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + VolumeInfo destData = Mockito.mock(VolumeInfo.class); + Host host = Mockito.mock(Host.class); + String srcVolumePath = "bec0ba7700000007:vol-11-6aef-10ee"; + String destVolumePath = "01b332b300000007:vol-11-b9e2-10ee"; + + when(srcData.getId()).thenReturn(1L); + when(srcData.getPoolId()).thenReturn(1L); + when(destData.getId()).thenReturn(1L); + + when(srcData.getPath()).thenReturn(srcVolumePath); + when(srcData.getFolder()).thenReturn("921c364500000007"); + DataStore destStore = Mockito.mock(DataStore.class); + when(destStore.getId()).thenReturn(2L); + when(destData.getDataStore()).thenReturn(destStore); + doNothing().when(scaleIOPrimaryDataStoreDriver).revokeAccess(any(), any(), any()); + + ScaleIOGatewayClient client = Mockito.mock(ScaleIOGatewayClient.class); + doReturn(client).when(scaleIOPrimaryDataStoreDriver) + .getScaleIOClient(any()); + when(client.deleteVolume(any())).thenReturn(false); + + VolumeVO volume = new VolumeVO("root", 1L, 1L, 1L, 1L, 1L, "root", "root", Storage.ProvisioningType.THIN, 1, null, null, "root", Volume.Type.ROOT); + when(volumeDao.findById(1L)).thenReturn(volume); + when(volumeDao.update(1L, volume)).thenReturn(true); + + scaleIOPrimaryDataStoreDriver.revertBlockCopyVolumeOperations(srcData, destData, host, destVolumePath); + + Assert.assertEquals(srcVolumePath, volume.get_iScsiName()); + Assert.assertEquals(srcVolumePath, volume.getPath()); + Assert.assertEquals("921c364500000007", volume.getFolder()); + } + + @Test + public void deleteSourceVolumeSuccessScenarioAfterSuccessfulBlockCopy() throws Exception { + // Either Volume deletion success or failure method should complete + + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + Host host = Mockito.mock(Host.class); + String srcVolumePath = "bec0ba7700000007:vol-11-6aef-10ee"; + + DataStore srcStore = Mockito.mock(DataStore.class); + DataTO volumeTO = Mockito.mock(DataTO.class); + when(srcData.getDataStore()).thenReturn(srcStore); + when(srcData.getTO()).thenReturn(volumeTO); + when(volumeTO.getPath()).thenReturn(srcVolumePath); + doNothing().when(scaleIOPrimaryDataStoreDriver).revokeAccess(any(), any(), any()); + + ScaleIOGatewayClient client = Mockito.mock(ScaleIOGatewayClient.class); + doReturn(client).when(scaleIOPrimaryDataStoreDriver) + .getScaleIOClient(any()); + when(client.deleteVolume(any())).thenReturn(true); + + scaleIOPrimaryDataStoreDriver.deleteSourceVolumeAfterSuccessfulBlockCopy(srcData, host); + } + + @Test + public void deleteSourceVolumeFailureScenarioAfterSuccessfulBlockCopy() throws Exception { + // Either Volume deletion success or failure method should complete + + VolumeInfo srcData = Mockito.mock(VolumeInfo.class); + Host host = Mockito.mock(Host.class); + String srcVolumePath = "bec0ba7700000007:vol-11-6aef-10ee"; + + DataStore srcStore = Mockito.mock(DataStore.class); + DataTO volumeTO = Mockito.mock(DataTO.class); + when(srcData.getDataStore()).thenReturn(srcStore); + when(srcData.getTO()).thenReturn(volumeTO); + when(volumeTO.getPath()).thenReturn(srcVolumePath); + doNothing().when(scaleIOPrimaryDataStoreDriver).revokeAccess(any(), any(), any()); + + ScaleIOGatewayClient client = Mockito.mock(ScaleIOGatewayClient.class); + doReturn(client).when(scaleIOPrimaryDataStoreDriver) + .getScaleIOClient(any()); + when(client.deleteVolume(any())).thenReturn(false); + + scaleIOPrimaryDataStoreDriver.deleteSourceVolumeAfterSuccessfulBlockCopy(srcData, host); + } } \ No newline at end of file From 691bf6e473d375fbe4c5c5ee3754bd649497344e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 31 May 2023 17:00:51 +0530 Subject: [PATCH 16/31] Fixed offline volume migration case and allowed encrypted volume migration --- .../driver/ScaleIOPrimaryDataStoreDriver.java | 54 +++++++++++++++++-- .../ScaleIOPrimaryDataStoreDriverTest.java | 8 +-- .../cloud/storage/VolumeApiServiceImpl.java | 38 ------------- .../storage/VolumeApiServiceImplTest.java | 2 - 4 files changed, 55 insertions(+), 47 deletions(-) diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 3d846de084d8..783377ea5035 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -258,6 +258,27 @@ public void revokeAccess(DataObject dataObject, Host host, DataStore dataStore) } } + public void revokeVolumeAccess(String volumePath, Host host, DataStore dataStore) { + if (host == null) { + LOGGER.warn("Declining to revoke access to PowerFlex volume when a host is not provided"); + return; + } + + try { + LOGGER.debug("Revoking access for PowerFlex volume: " + volumePath); + + final String sdcId = getConnectedSdc(dataStore.getId(), host.getId()); + if (StringUtils.isBlank(sdcId)) { + throw new CloudRuntimeException("Unable to revoke access for volume: " + volumePath + ", no Sdc connected with host ip: " + host.getPrivateIpAddress()); + } + + final ScaleIOGatewayClient client = getScaleIOClient(dataStore.getId()); + client.unmapVolumeFromSdc(ScaleIOUtil.getVolumePath(volumePath), sdcId); + } catch (Exception e) { + LOGGER.warn("Failed to revoke access due to: " + e.getMessage(), e); + } + } + private void revokeAccess(DataObject dataObject, EndPoint ep, DataStore dataStore) { Host host = hostDao.findById(ep.getId()); revokeAccess(dataObject, host, dataStore); @@ -687,7 +708,12 @@ public void copyAsync(DataObject srcData, DataObject destData, Host destHost, As if (isSameScaleIOStorageInstance(srcStore, destStore)) { answer = migrateVolume(srcData, destData); } else { - answer = copyVolume(srcData, destData); + String vmName = ((VolumeInfo) srcData).getAttachedVmName(); + if (vmName == null || !vmInstanceDao.findVMByInstanceName(vmName).getState().equals(VirtualMachine.State.Running)) { + answer = copyOfflineVolume(srcData, destData, destHost); + } else { + answer = liveMigrateVolume(srcData, destData); + } } } else { errMsg = "Unsupported copy operation from src object: (" + srcData.getType() + ", " + srcData.getDataStore() + "), dest object: (" @@ -761,7 +787,29 @@ private Answer copyTemplateToVolume(DataObject srcData, DataObject destData, Hos return answer; } - public Answer copyVolume(DataObject srcData, DataObject destData) { + private Answer copyOfflineVolume(DataObject srcData, DataObject destData, Host destHost) { + // Copy PowerFlex/ScaleIO volume + LOGGER.debug(String.format("Initiating copy from PowerFlex template volume on host %s", destHost != null ? destHost.getId() : "")); + String value = configDao.getValue(Config.CopyVolumeWait.key()); + int copyVolumeWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); + + CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), copyVolumeWait, VirtualMachineManager.ExecuteInSequence.value()); + + Answer answer = null; + boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData); + EndPoint ep = destHost != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(destHost) : selector.select(srcData, encryptionRequired); + if (ep == null) { + String errorMsg = String.format("No remote endpoint to send command, unable to find a valid endpoint. Requires encryption support: %s", encryptionRequired); + LOGGER.error(errorMsg); + answer = new Answer(cmd, false, errorMsg); + } else { + answer = ep.sendMessage(cmd); + } + + return answer; + } + + public Answer liveMigrateVolume(DataObject srcData, DataObject destData) { // Volume migration across different PowerFlex/ScaleIO clusters final long srcVolumeId = srcData.getId(); DataStore srcStore = srcData.getDataStore(); @@ -887,7 +935,7 @@ public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destDa public void deleteSourceVolumeAfterSuccessfulBlockCopy(DataObject srcData, Host host) { DataStore srcStore = srcData.getDataStore(); String srcVolumePath = srcData.getTO().getPath(); - revokeAccess(srcData, host, srcData.getDataStore()); + revokeVolumeAccess(srcVolumePath, host, srcData.getDataStore()); String errMsg; try { String scaleIOVolumeId = ScaleIOUtil.getVolumePath(srcVolumePath); diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java index 643bf9165b7c..46f389b453f4 100644 --- a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java +++ b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java @@ -203,7 +203,7 @@ public void testMigrateVolumeWithinSameScaleIOClusterSuccess() throws Exception Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver) .deleteSourceVolumeAfterSuccessfulBlockCopy(any(), any()); - Answer answer = scaleIOPrimaryDataStoreDriver.copyVolume(srcData, destData); + Answer answer = scaleIOPrimaryDataStoreDriver.liveMigrateVolume(srcData, destData); Assert.assertTrue(answer.getResult()); } @@ -243,7 +243,7 @@ public void testMigrateVolumeWithinSameScaleIOClusterFailure() throws Exception Mockito.doNothing().when(scaleIOPrimaryDataStoreDriver) .revertBlockCopyVolumeOperations(any(), any(), any(), any()); - Answer answer = scaleIOPrimaryDataStoreDriver.copyVolume(srcData, destData); + Answer answer = scaleIOPrimaryDataStoreDriver.liveMigrateVolume(srcData, destData); Assert.assertFalse(answer.getResult()); } @@ -422,7 +422,7 @@ public void deleteSourceVolumeSuccessScenarioAfterSuccessfulBlockCopy() throws E when(srcData.getDataStore()).thenReturn(srcStore); when(srcData.getTO()).thenReturn(volumeTO); when(volumeTO.getPath()).thenReturn(srcVolumePath); - doNothing().when(scaleIOPrimaryDataStoreDriver).revokeAccess(any(), any(), any()); + doNothing().when(scaleIOPrimaryDataStoreDriver).revokeVolumeAccess(any(), any(), any()); ScaleIOGatewayClient client = Mockito.mock(ScaleIOGatewayClient.class); doReturn(client).when(scaleIOPrimaryDataStoreDriver) @@ -445,7 +445,7 @@ public void deleteSourceVolumeFailureScenarioAfterSuccessfulBlockCopy() throws E when(srcData.getDataStore()).thenReturn(srcStore); when(srcData.getTO()).thenReturn(volumeTO); when(volumeTO.getPath()).thenReturn(srcVolumePath); - doNothing().when(scaleIOPrimaryDataStoreDriver).revokeAccess(any(), any(), any()); + doNothing().when(scaleIOPrimaryDataStoreDriver).revokeVolumeAccess(any(), any(), any()); ScaleIOGatewayClient client = Mockito.mock(ScaleIOGatewayClient.class); doReturn(client).when(scaleIOPrimaryDataStoreDriver) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 7c2e1db6f23c..8de1aaba20b4 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -92,7 +92,6 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; @@ -2970,10 +2969,6 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { vm = _vmInstanceDao.findById(instanceId); } - if (vol.getPassphraseId() != null) { - throw new InvalidParameterValueException("Migration of encrypted volumes is unsupported"); - } - // Check that Vm to which this volume is attached does not have VM Snapshots // OfflineVmwareMigration: consider if this is needed and desirable if (vm != null && _vmSnapshotDao.findByVm(vm.getId()).size() > 0) { @@ -3023,14 +3018,6 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { } } - // Offline volume migration check for scaleIO volumes across scaleio clusters - if (vm == null || !State.Running.equals(vm.getState())) { - StoragePoolVO sourceStoragePoolVO = _storagePoolDao.findById(vol.getPoolId()); - if (sourceStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && isScaleIOVolumeOnDifferentScaleIOStorageInstances(vol.getPoolId(), storagePoolId)) { - throw new InvalidParameterValueException("Volume needs to be attached to a running VM to move across ScaleIO storages in different ScaleIO clusters"); - } - } - if (vm != null && HypervisorType.VMware.equals(vm.getHypervisorType()) && State.Stopped.equals(vm.getState())) { @@ -3164,31 +3151,6 @@ public Volume migrateVolume(MigrateVolumeCmd cmd) { return orchestrateMigrateVolume(vol, destPool, liveMigrateVolume, newDiskOffering); } - protected boolean isScaleIOVolumeOnDifferentScaleIOStorageInstances(long srcPoolId, long destPoolId) { - String SCALEIO_STORAGE_POOL_SYSTEM_ID = "powerflex.storagepool.system.id"; - String srcPoolSystemId = null; - StoragePoolDetailVO srcPoolSystemIdDetail = storagePoolDetailsDao.findDetail(srcPoolId, SCALEIO_STORAGE_POOL_SYSTEM_ID); - if (srcPoolSystemIdDetail != null) { - srcPoolSystemId = srcPoolSystemIdDetail.getValue(); - } - - String destPoolSystemId = null; - StoragePoolDetailVO destPoolSystemIdDetail = storagePoolDetailsDao.findDetail(destPoolId, SCALEIO_STORAGE_POOL_SYSTEM_ID); - if (destPoolSystemIdDetail != null) { - destPoolSystemId = destPoolSystemIdDetail.getValue(); - } - - if (StringUtils.isAnyEmpty(srcPoolSystemId, destPoolSystemId)) { - throw new CloudRuntimeException("Failed to validate PowerFlex pools compatibility for migration as storage instance details are not available"); - } - - if (!srcPoolSystemId.equals(destPoolSystemId)) { - return true; - } - - return false; - } - private boolean isSourceOrDestNotOnStorPool(StoragePoolVO storagePoolVO, StoragePoolVO destinationStoragePoolVo) { return storagePoolVO.getPoolType() != Storage.StoragePoolType.StorPool || destinationStoragePoolVo.getPoolType() != Storage.StoragePoolType.StorPool; diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index a0ece775c928..71a4b76650e6 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -54,7 +54,6 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; @@ -115,7 +114,6 @@ import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; From 8558e047e33184240a3244eca3b5560f8fb4fcb9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 11 Apr 2023 19:28:31 +0530 Subject: [PATCH 17/31] Added more integration tests --- .../plugins/scaleio/test_scaleio_volumes.py | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) diff --git a/test/integration/plugins/scaleio/test_scaleio_volumes.py b/test/integration/plugins/scaleio/test_scaleio_volumes.py index c67f838297bf..d6ba569f90f0 100644 --- a/test/integration/plugins/scaleio/test_scaleio_volumes.py +++ b/test/integration/plugins/scaleio/test_scaleio_volumes.py @@ -1144,6 +1144,249 @@ def test_10_migrate_volume_to_distinct_instance_pool(self): test_virtual_machine.delete(self.apiClient, True) + @attr(tags=['advanced', 'migration'], required_hardware=False) + def test_11_live_migrate_volume_to_same_instance_pool(self): + '''Migrate volume to the same instance pool''' + + if not TestData.migrationTests: + self.skipTest("Volume migration tests not enabled, skipping test") + + ####################################### + # STEP 1: Create VM and Start VM # + ####################################### + + test_virtual_machine = VirtualMachine.create( + self.apiClient, + self.testdata[TestData.virtualMachine3], + accountid=self.account.name, + zoneid=self.zone.id, + serviceofferingid=self.compute_offering.id, + templateid=self.template.id, + domainid=self.domain.id, + startvm=False + ) + + TestScaleIOVolumes._start_vm(test_virtual_machine) + + ####################################### + # STEP 2: Create vol and attach to VM # + ####################################### + + new_volume = Volume.create( + self.apiClient, + self.testdata[TestData.volume_3], + account=self.account.name, + domainid=self.domain.id, + zoneid=self.zone.id, + diskofferingid=self.disk_offering_same_inst.id + ) + + volume_to_delete_later = new_volume + + new_volume = test_virtual_machine.attach_volume( + self.apiClient, + new_volume + ) + + vm = self._get_vm(test_virtual_machine.id) + + self.assertEqual( + new_volume.virtualmachineid, + vm.id, + "Check if attached to virtual machine" + ) + + self.assertEqual( + vm.state.lower(), + 'running', + str(vm.state) + ) + + ####################################### + # STEP 3: Migrate volume # + ####################################### + + pools = StoragePool.listForMigration( + self.apiClient, + id=new_volume.id + ) + + if not pools: + self.skipTest("No suitable storage pools found for volume migration, skipping test") + + self.assertEqual( + validateList(pools)[0], + PASS, + "Invalid pool response from findStoragePoolsForMigration API" + ) + + pool = pools[0] + self.debug("Migrating Volume-ID: %s to Same Instance Pool: %s" % (new_volume.id, pool.id)) + + try: + Volume.migrate( + self.apiClient, + volumeid=new_volume.id, + storageid=pool.id + ) + except Exception as e: + self.fail("Volume migration failed with error %s" % e) + + ####################################### + # STEP 4: Detach and delete volume # + ####################################### + + new_volume = test_virtual_machine.detach_volume( + self.apiClient, + new_volume + ) + + self.assertEqual( + new_volume.virtualmachineid, + None, + "Check if attached to virtual machine" + ) + + volume_to_delete_later.delete(self.apiClient) + + list_volumes_response = list_volumes( + self.apiClient, + id=new_volume.id + ) + + self.assertEqual( + list_volumes_response, + None, + "Check volume was deleted" + ) + + ####################################### + # STEP 4: Delete VM # + ####################################### + + test_virtual_machine.delete(self.apiClient, True) + + @attr(tags=['advanced', 'migration'], required_hardware=False) + def test_12_migrate_volume_to_distinct_instance_pool(self): + '''Migrate volume to distinct instance pool''' + + if not TestData.migrationTests: + self.skipTest("Volume migration tests not enabled, skipping test") + + ####################################### + # STEP 1: Create VM and Start VM # + ####################################### + + test_virtual_machine = VirtualMachine.create( + self.apiClient, + self.testdata[TestData.virtualMachine4], + accountid=self.account.name, + zoneid=self.zone.id, + serviceofferingid=self.compute_offering.id, + templateid=self.template.id, + domainid=self.domain.id, + startvm=False + ) + + TestScaleIOVolumes._start_vm(test_virtual_machine) + + ####################################### + # STEP 2: Create vol and attach to VM # + ####################################### + + new_volume = Volume.create( + self.apiClient, + self.testdata[TestData.volume_4], + account=self.account.name, + domainid=self.domain.id, + zoneid=self.zone.id, + diskofferingid=self.disk_offering_distinct_inst.id + ) + + volume_to_delete_later = new_volume + + new_volume = test_virtual_machine.attach_volume( + self.apiClient, + new_volume + ) + + vm = self._get_vm(test_virtual_machine.id) + + self.assertEqual( + new_volume.virtualmachineid, + vm.id, + "Check if attached to virtual machine" + ) + + self.assertEqual( + vm.state.lower(), + 'running', + str(vm.state) + ) + + ####################################### + # STEP 3: Migrate volume # + ####################################### + + pools = StoragePool.listForMigration( + self.apiClient, + id=new_volume.id + ) + + if not pools: + self.skipTest("No suitable storage pools found for volume migration, skipping test") + + self.assertEqual( + validateList(pools)[0], + PASS, + "Invalid pool response from findStoragePoolsForMigration API" + ) + + pool = pools[0] + self.debug("Migrating Volume-ID: %s to Distinct Instance Pool: %s" % (new_volume.id, pool.id)) + + try: + Volume.migrate( + self.apiClient, + volumeid=new_volume.id, + storageid=pool.id + ) + except Exception as e: + self.fail("Volume migration failed with error %s" % e) + + ####################################### + # STEP 4: Detach and delete volume # + ####################################### + + new_volume = test_virtual_machine.detach_volume( + self.apiClient, + new_volume + ) + + self.assertEqual( + new_volume.virtualmachineid, + None, + "Check if attached to virtual machine" + ) + + volume_to_delete_later.delete(self.apiClient) + + list_volumes_response = list_volumes( + self.apiClient, + id=new_volume.id + ) + + self.assertEqual( + list_volumes_response, + None, + "Check volume was deleted" + ) + + ####################################### + # STEP 4: Delete VM # + ####################################### + + test_virtual_machine.delete(self.apiClient, True) def _create_vm_using_template_and_destroy_vm(self, template): vm_name = "VM-%d" % random.randint(0, 100) From e300eeed70492e1dc5aa6dbd613363e16b9d1854 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 5 May 2023 17:17:55 +0530 Subject: [PATCH 18/31] Support for migration of encrypted volumes across different scaleio clusters --- .../LibvirtMigrateVolumeCommandWrapper.java | 101 +++++++++++++----- .../driver/ScaleIOPrimaryDataStoreDriver.java | 8 +- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 2e7a4192c569..7630cb9d3ba7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -23,10 +23,7 @@ import com.cloud.agent.api.storage.MigrateVolumeAnswer; import com.cloud.agent.api.storage.MigrateVolumeCommand; import com.cloud.agent.api.to.DiskTO; -import com.cloud.exception.InternalErrorException; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; -import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; @@ -34,8 +31,10 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage; +import java.io.ByteArrayInputStream; import java.io.File; -import java.util.List; +import java.io.IOException; +import java.io.StringWriter; import java.util.Map; import java.util.UUID; @@ -43,6 +42,7 @@ import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.Domain; @@ -51,6 +51,20 @@ import org.libvirt.TypedParameter; import org.libvirt.TypedUlongParameter; import org.libvirt.LibvirtException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; @ResourceWrapper(handles = MigrateVolumeCommand.class) public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { @@ -109,15 +123,15 @@ private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand co KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid()); pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); - LibvirtVMDef.DiskDef diskdef = generateDestinationDiskDefinition(dm, srcVolumeId, srcPath, diskFilePath); - destDiskLabel = diskdef.getDiskLabel(); + String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath); + destDiskLabel = generateDestinationDiskLabel(diskdef); TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); TypedParameter[] parameters = new TypedParameter[1]; parameters[0] = parameter; - dm.blockCopy(destDiskLabel, diskdef.toString(), parameters, Domain.BlockCopyFlags.REUSE_EXT); - LOGGER.info(String.format("Block copy has started for the volume %s : %s ", diskdef.getDiskLabel(), srcPath)); + dm.blockCopy(destDiskLabel, diskdef, parameters, Domain.BlockCopyFlags.REUSE_EXT); + LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath)); return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath); @@ -149,7 +163,7 @@ private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Do while (waitTimeInSec > 0) { DomainBlockJobInfo blockJobInfo = dm.getBlockJobInfo(diskLabel, 0); if (blockJobInfo != null) { - LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% ", diskLabel, srcPath, 100 * (blockJobInfo.cur / blockJobInfo.end))); + LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end)); if (blockJobInfo.cur == blockJobInfo.end) { LOGGER.debug(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); @@ -182,26 +196,65 @@ private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Do return new MigrateVolumeAnswer(command, true, null, destPath); } - private LibvirtVMDef.DiskDef generateDestinationDiskDefinition(Domain dm, String srcVolumeId, String srcPath, String diskFilePath) throws InternalErrorException, LibvirtException { - final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + private String generateDestinationDiskLabel(String diskXml) throws ParserConfigurationException, IOException, SAXException { + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(diskXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + Element disk = doc.getDocumentElement(); + String diskLabel = getAttrValue("target", "dev", disk); + + return diskLabel; + } + + private String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { final String domXml = dm.getXMLDesc(0); - parser.parseDomainXML(domXml); - - List disks = parser.getDisks(); - LibvirtVMDef.DiskDef diskdef = null; - for (final LibvirtVMDef.DiskDef disk : disks) { - final String file = disk.getDiskPath(); - if (file != null && file.contains(srcVolumeId)) { - diskdef = disk; - break; + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(new ByteArrayInputStream(domXml.getBytes("UTF-8"))); + doc.getDocumentElement().normalize(); + + NodeList disks = doc.getElementsByTagName("disk"); + + for (int i = 0; i < disks.getLength(); i++) { + Element disk = (Element)disks.item(i); + String type = disk.getAttribute("type"); + if (!type.equalsIgnoreCase("network")) { + String diskDev = getAttrValue("source", "dev", disk); + if (StringUtils.isNotEmpty(diskDev) && diskDev.contains(srcVolumeId)) { + setAttrValue("source", "dev", diskFilePath, disk); + StringWriter diskSection = new StringWriter(); + Transformer xformer = TransformerFactory.newInstance().newTransformer(); + xformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + xformer.transform(new DOMSource(disk), new StreamResult(diskSection)); + + return diskSection.toString(); + } } } - if (diskdef == null) { - throw new InternalErrorException("disk: " + srcPath + " is not attached before"); + + return null; + } + + private static String getAttrValue(String tag, String attr, Element eElement) { + NodeList tagNode = eElement.getElementsByTagName(tag); + if (tagNode.getLength() == 0) { + return null; } - diskdef.setDiskPath(diskFilePath); + Element node = (Element)tagNode.item(0); + return node.getAttribute(attr); + } - return diskdef; + private static void setAttrValue(String tag, String attr, String newValue, Element eElement) { + NodeList tagNode = eElement.getElementsByTagName(tag); + if (tagNode.getLength() == 0) { + return; + } + Element node = (Element)tagNode.item(0); + node.setAttribute(attr, newValue); } private MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 783377ea5035..0aa1466e3329 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -470,6 +470,10 @@ public void revertSnapshot(SnapshotInfo snapshot, SnapshotInfo snapshotOnPrimary } public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId) { + return createVolume(volumeInfo, storagePoolId, false); + } + + public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId, boolean migrationInvolved) { LOGGER.debug("Creating PowerFlex volume"); StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); @@ -514,7 +518,7 @@ public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId CreateObjectAnswer answer = new CreateObjectAnswer(createdObject.getTO()); // if volume needs to be set up with encryption, do it now if it's not a root disk (which gets done during template copy) - if (anyVolumeRequiresEncryption(volumeInfo) && !volumeInfo.getVolumeType().equals(Volume.Type.ROOT)) { + if (anyVolumeRequiresEncryption(volumeInfo) && (!volumeInfo.getVolumeType().equals(Volume.Type.ROOT) || migrationInvolved)) { LOGGER.debug(String.format("Setting up encryption for volume %s", volumeInfo.getId())); VolumeObjectTO prepVolume = (VolumeObjectTO) createdObject.getTO(); prepVolume.setPath(volumePath); @@ -826,7 +830,7 @@ public Answer liveMigrateVolume(DataObject srcData, DataObject destData) { Answer answer = null; try { - CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId()); + CreateObjectAnswer createAnswer = createVolume((VolumeInfo) destData, destStore.getId(), true); destVolumePath = createAnswer.getData().getPath(); destVolTO.setPath(destVolumePath); From 06db2a6441a348f68089383bc089ef2b34f91127 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 31 May 2023 17:05:49 +0530 Subject: [PATCH 19/31] Fix UI notifications for migrate volume --- .../resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java | 4 ++-- ui/src/views/storage/MigrateVolume.vue | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 7630cb9d3ba7..0e64d65ba507 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -165,12 +165,12 @@ private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Do if (blockJobInfo != null) { LOGGER.debug(String.format("Volume %s : %s block copy progress: %s%% current value:%s end value:%s", diskLabel, srcPath, (blockJobInfo.end == 0)? 0 : 100*(blockJobInfo.cur / (double) blockJobInfo.end), blockJobInfo.cur, blockJobInfo.end)); if (blockJobInfo.cur == blockJobInfo.end) { - LOGGER.debug(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); + LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); break; } } else { - LOGGER.debug("Failed to get the block copy status, trying to abort the job"); + LOGGER.info("Failed to get the block copy status, trying to abort the job"); dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.ASYNC); } waitTimeInSec--; diff --git a/ui/src/views/storage/MigrateVolume.vue b/ui/src/views/storage/MigrateVolume.vue index 62bed7066089..604aae30c498 100644 --- a/ui/src/views/storage/MigrateVolume.vue +++ b/ui/src/views/storage/MigrateVolume.vue @@ -196,6 +196,8 @@ export default { title: this.$t('label.migrate.volume'), description: this.resource.id, jobId: response.migratevolumeresponse.jobid, + description: this.resource.name, + title: this.$t('label.migrate.volume'), successMessage: this.$t('message.success.migrate.volume'), errorMessage: this.$t('message.migrate.volume.failed'), errorMethod: () => { From d0b794bb8f464d0186ddf39e742bf74494551e8f Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 29 May 2023 17:12:45 +0530 Subject: [PATCH 20/31] Data volume offline migration: save encryption details to destination volume entry --- .../apache/cloudstack/storage/volume/VolumeServiceImpl.java | 4 ++++ .../com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java | 1 + 2 files changed, 5 insertions(+) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 48de0eb016be..ffc12b98c849 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1648,6 +1648,10 @@ protected VolumeVO duplicateVolumeOnAnotherStorage(Volume volume, StoragePool po newVol.setPoolType(pool.getPoolType()); newVol.setLastPoolId(lastPoolId); newVol.setPodId(pool.getPodId()); + if (volume.getPassphraseId() != null) { + newVol.setPassphraseId(volume.getPassphraseId()); + newVol.setEncryptFormat(volume.getEncryptFormat()); + } return volDao.persist(newVol); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index cae872e287f2..80d18a737eeb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -2467,6 +2467,7 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { String path = destPrimaryStore.isManaged() ? destVolumeName : destVolumePath + File.separator + destVolumeName; newVol.setPath(path); newVol.setFormat(destFormat); + newVol.setEncryptFormat(destVol.getEncryptFormat()); return new CopyCmdAnswer(newVol); } catch (final CloudRuntimeException e) { s_logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e); From d87d9da1d28c51c38dbb29a0be3b35417943f7b9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 30 May 2023 10:17:53 +0530 Subject: [PATCH 21/31] Offline storage migration for scaleio encrypted volumes --- .../cloud/hypervisor/kvm/storage/KVMStorageProcessor.java | 8 +++++++- .../datastore/driver/ScaleIOPrimaryDataStoreDriver.java | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 80d18a737eeb..a8a7d6f56944 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -37,6 +37,7 @@ import javax.naming.ConfigurationException; import com.cloud.storage.ScopeType; +import com.cloud.storage.Volume; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; @@ -2448,7 +2449,12 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { destPool = storagePoolMgr.getStoragePool(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid()); try { - storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + if (srcVol.getPassphrase() != null && srcVol.getVolumeType().equals(Volume.Type.ROOT)) { + volume.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds(), srcVol.getPassphrase(), destVol.getPassphrase(), srcVol.getProvisioningType()); + } else { + storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + } } catch (Exception e) { // Any exceptions while copying the disk, should send failed answer with the error message String errMsg = String.format("Failed to copy volume: %s to dest storage: %s, due to %s", srcVol.getName(), destPrimaryStore.getName(), e.toString()); s_logger.debug(errMsg, e); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 0aa1466e3329..f59734998712 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -504,7 +504,11 @@ public CreateObjectAnswer createVolume(VolumeInfo volumeInfo, long storagePoolId volume.setFolder(scaleIOVolume.getVtreeId()); volume.setSize(scaleIOVolume.getSizeInKb() * 1024); volume.setPoolType(Storage.StoragePoolType.PowerFlex); - volume.setFormat(Storage.ImageFormat.RAW); + if (volumeInfo.getVolumeType().equals(Volume.Type.ROOT)) { + volume.setFormat(volumeInfo.getFormat()); + } else { + volume.setFormat(Storage.ImageFormat.RAW); + } volume.setPoolId(storagePoolId); VolumeObject createdObject = VolumeObject.getVolumeObject(volumeInfo.getDataStore(), volume); createdObject.update(); From ffaf2bef3afb99e7abef1d5ca2b7b9e7c3ce2dd8 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 31 May 2023 17:09:21 +0530 Subject: [PATCH 22/31] Allow multiple Volumes to be migrated with migrateVirtualMachineWithVolume API --- .../src/main/java/com/cloud/vm/UserVmManagerImpl.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 3f73b620c389..f0aed627f15c 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -6270,16 +6270,6 @@ private VMInstanceVO preVmStorageMigrationCheck(Long vmId) { + " hypervisors: [%s].", hypervisorType, HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS)); } - List vols = _volsDao.findByInstance(vm.getId()); - if (vols.size() > 1) { - // OffLineVmwareMigration: data disks are not permitted, here! - if (vols.size() > 1 && - // OffLineVmwareMigration: allow multiple disks for vmware - !HypervisorType.VMware.equals(hypervisorType)) { - throw new InvalidParameterValueException("Data disks attached to the vm, can not migrate. Need to detach data disks first"); - } - } - // Check that Vm does not have VM Snapshots if (_vmSnapshotDao.findByVm(vmId).size() > 0) { throw new InvalidParameterValueException("VM's disk cannot be migrated, please remove all the VM Snapshots for this VM"); From c3c45c971bea2e2c8135b9c0634ccadf05fe0ca1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 31 May 2023 17:14:31 +0530 Subject: [PATCH 23/31] Removed unused unittests --- .../storage/VolumeApiServiceImplTest.java | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 71a4b76650e6..8e246340dcb5 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -1567,55 +1567,4 @@ public void testIsSendCommandForVmVolumeAttachDetachKVMHost() { Assert.assertFalse(volumeApiServiceImpl.isSendCommandForVmVolumeAttachDetach(host, Mockito.mock(StoragePoolVO.class))); } - @Test - public void testVolumeOnDifferentScaleIOStorageInstance() { - StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); - String srcPoolSystemId = "610204d03e3ad60f"; - when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); - - StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); - String destPoolSystemId = "7332760565f6340f"; - when(destPoolSystemIdDetail.getValue()).thenReturn(destPoolSystemId); - - when(storagePoolDetailsDao.findDetail(1L,"powerflex.storagepool.system.id")).thenReturn(srcPoolSystemIdDetail); - when(storagePoolDetailsDao.findDetail(2L,"powerflex.storagepool.system.id")).thenReturn(destPoolSystemIdDetail); - - boolean result = volumeApiServiceImpl.isScaleIOVolumeOnDifferentScaleIOStorageInstances(1L, 2L); - - Assert.assertTrue(result); - } - - @Test - public void testVolumeOnSameScaleIOStorageInstance() { - StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); - String srcPoolSystemId = "610204d03e3ad60f"; - when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); - - StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); - String destPoolSystemId = "610204d03e3ad60f"; - when(destPoolSystemIdDetail.getValue()).thenReturn(destPoolSystemId); - - when(storagePoolDetailsDao.findDetail(1L,"powerflex.storagepool.system.id")).thenReturn(srcPoolSystemIdDetail); - when(storagePoolDetailsDao.findDetail(2L,"powerflex.storagepool.system.id")).thenReturn(destPoolSystemIdDetail); - - boolean result = volumeApiServiceImpl.isScaleIOVolumeOnDifferentScaleIOStorageInstances(1L, 2L); - - Assert.assertFalse(result); - } - - @Test (expected = CloudRuntimeException.class) - public void testCheckVolumeOnDifferentScaleIOStorageInstanceSystemIdShouldNotBeNull() { - StoragePoolDetailVO srcPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); - String srcPoolSystemId = "610204d03e3ad60f"; - when(srcPoolSystemIdDetail.getValue()).thenReturn(srcPoolSystemId); - - StoragePoolDetailVO destPoolSystemIdDetail = Mockito.mock(StoragePoolDetailVO.class); - when(destPoolSystemIdDetail.getValue()).thenReturn(null); - - when(storagePoolDetailsDao.findDetail(1L,"powerflex.storagepool.system.id")).thenReturn(srcPoolSystemIdDetail); - when(storagePoolDetailsDao.findDetail(2L,"powerflex.storagepool.system.id")).thenReturn(destPoolSystemIdDetail); - - volumeApiServiceImpl.isScaleIOVolumeOnDifferentScaleIOStorageInstances(1L, 2L); - } - } From 417b8600f643992ffd5a2dcce1df675234a490e0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 1 Jun 2023 09:54:39 +0530 Subject: [PATCH 24/31] Removed duplicate keys in migrate volume vue file --- ui/src/views/storage/MigrateVolume.vue | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/src/views/storage/MigrateVolume.vue b/ui/src/views/storage/MigrateVolume.vue index 604aae30c498..62bed7066089 100644 --- a/ui/src/views/storage/MigrateVolume.vue +++ b/ui/src/views/storage/MigrateVolume.vue @@ -196,8 +196,6 @@ export default { title: this.$t('label.migrate.volume'), description: this.resource.id, jobId: response.migratevolumeresponse.jobid, - description: this.resource.name, - title: this.$t('label.migrate.volume'), successMessage: this.$t('message.success.migrate.volume'), errorMessage: this.$t('message.migrate.volume.failed'), errorMethod: () => { From 683fefdb2289864e0ae35f6e81a2aaf6dc33dbf6 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 6 Jun 2023 09:41:51 +0530 Subject: [PATCH 25/31] Fix Unit tests --- .../driver/client/ScaleIOPrimaryDataStoreDriverTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java index 46f389b453f4..ee1513d68cb1 100644 --- a/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java +++ b/plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/client/ScaleIOPrimaryDataStoreDriverTest.java @@ -186,7 +186,7 @@ public void testMigrateVolumeWithinSameScaleIOClusterSuccess() throws Exception DataTO dataTO = Mockito.mock(DataTO.class); CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO); - doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L); + doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true); when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee"); doReturn(true).when(scaleIOPrimaryDataStoreDriver) .grantAccess(any(), any(), any()); @@ -231,7 +231,7 @@ public void testMigrateVolumeWithinSameScaleIOClusterFailure() throws Exception DataTO dataTO = Mockito.mock(DataTO.class); CreateObjectAnswer createAnswer = new CreateObjectAnswer(dataTO); - doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L); + doReturn(createAnswer).when(scaleIOPrimaryDataStoreDriver).createVolume(destData, 2L, true); when(dataTO.getPath()).thenReturn("bec0ba7700000007:vol-11-6aef-10ee"); doReturn(true).when(scaleIOPrimaryDataStoreDriver) .grantAccess(any(), any(), any()); From eea8b2de5a1a7259aecd01b4dbf1fb178ca50783 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 6 Jun 2023 09:19:42 +0530 Subject: [PATCH 26/31] Add volume secrets if does not exists during volume migrations. secrets are getting cleared on package upgrades. --- .../resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 0e64d65ba507..acd7abb48cb6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -126,6 +126,11 @@ private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand co String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath); destDiskLabel = generateDestinationDiskLabel(diskdef); + if (destVolumeObjectTO.getPassphrase() != null) { + libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase()); + libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); + } + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); TypedParameter[] parameters = new TypedParameter[1]; parameters[0] = parameter; From cc48055131c02fc64cd677ee96e520c957c4acd8 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 7 Jun 2023 12:24:03 +0530 Subject: [PATCH 27/31] Fix secret UUID for encrypted volume migration --- .../LibvirtMigrateVolumeCommandWrapper.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index acd7abb48cb6..d29e5306ed73 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -123,14 +123,16 @@ private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand co KVMStoragePool pool = storagePoolMgr.getStoragePool(spool.getPoolType(), spool.getUuid()); pool.connectPhysicalDisk(destVolumeObjectTO.getPath(), null); - String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath); - destDiskLabel = generateDestinationDiskLabel(diskdef); - + String srcSecretUUID = null; + String destSecretUUID = null; if (destVolumeObjectTO.getPassphrase() != null) { - libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase()); - libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); + srcSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, srcVolumeObjectTO.getPath(), srcVolumeObjectTO.getPassphrase()); + destSecretUUID = libvirtComputingResource.createLibvirtVolumeSecret(conn, destVolumeObjectTO.getPath(), destVolumeObjectTO.getPassphrase()); } + String diskdef = generateDestinationDiskXML(dm, srcVolumeId, diskFilePath, destSecretUUID); + destDiskLabel = generateDestinationDiskLabel(diskdef); + TypedUlongParameter parameter = new TypedUlongParameter("bandwidth", 0); TypedParameter[] parameters = new TypedParameter[1]; parameters[0] = parameter; @@ -138,7 +140,7 @@ private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand co dm.blockCopy(destDiskLabel, diskdef, parameters, Domain.BlockCopyFlags.REUSE_EXT); LOGGER.info(String.format("Block copy has started for the volume %s : %s ", destDiskLabel, srcPath)); - return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath); + return checkBlockJobStatus(command, dm, destDiskLabel, srcPath, destPath, libvirtComputingResource, conn, srcSecretUUID); } catch (Exception e) { String msg = "Migrate volume failed due to " + e.toString(); @@ -162,7 +164,7 @@ private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand co } } - private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath) throws LibvirtException { + private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found int waitTimeInSec = command.getWait(); while (waitTimeInSec > 0) { @@ -172,6 +174,7 @@ private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Do if (blockJobInfo.cur == blockJobInfo.end) { LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); + libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); break; } } else { @@ -214,7 +217,7 @@ private String generateDestinationDiskLabel(String diskXml) throws ParserConfigu return diskLabel; } - private String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { + private String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { final String domXml = dm.getXMLDesc(0); DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); @@ -231,6 +234,9 @@ private String generateDestinationDiskXML(Domain dm, String srcVolumeId, String String diskDev = getAttrValue("source", "dev", disk); if (StringUtils.isNotEmpty(diskDev) && diskDev.contains(srcVolumeId)) { setAttrValue("source", "dev", diskFilePath, disk); + if (StringUtils.isNotEmpty(destSecretUUID)) { + setAttrValue("secret", "uuid", destSecretUUID, disk); + } StringWriter diskSection = new StringWriter(); Transformer xformer = TransformerFactory.newInstance().newTransformer(); xformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); From e0623b08d966a9119ea05e4466a002882d8fea17 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 7 Jun 2023 14:45:57 +0530 Subject: [PATCH 28/31] Added a null check for secret before removing --- .../resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index d29e5306ed73..bc347b302aef 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -174,7 +174,9 @@ private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Do if (blockJobInfo.cur == blockJobInfo.end) { LOGGER.info(String.format("Block copy completed for the volume %s : %s", diskLabel, srcPath)); dm.blockJobAbort(diskLabel, Domain.BlockJobAbortFlags.PIVOT); - libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); + if (StringUtils.isNotEmpty(srcSecretUUID)) { + libvirtComputingResource.removeLibvirtVolumeSecret(conn, srcSecretUUID); + } break; } } else { From c3ff286404a87aff5b51ea10e5389438190fd5ba Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 9 Jun 2023 14:07:12 +0530 Subject: [PATCH 29/31] Added more unit tests --- .../vm/VirtualMachineManagerImplTest.java | 11 + .../LibvirtMigrateVolumeCommandWrapper.java | 10 +- ...ibvirtMigrateVolumeCommandWrapperTest.java | 388 ++++++++++++++++++ .../driver/ScaleIOPrimaryDataStoreDriver.java | 8 +- .../ScaleIOPrimaryDataStoreDriverTest.java | 76 +++- .../cloud/storage/VolumeApiServiceImpl.java | 6 +- .../storage/VolumeApiServiceImplTest.java | 4 - 7 files changed, 484 insertions(+), 19 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java rename plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/driver/{client => }/ScaleIOPrimaryDataStoreDriverTest.java (85%) diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index fd3dba6dbd50..6f40c5d3a3a6 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -377,6 +377,17 @@ public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentS Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId(); } + @Test + public void allowVolumeMigrationsForPowerFlexStorage() { + Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); + Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(storagePoolVoMock).getPoolType(); + + virtualMachineManagerImpl.executeManagedStorageChecksWhenTargetStoragePoolProvided(storagePoolVoMock, volumeVoMock, Mockito.mock(StoragePoolVO.class)); + + Mockito.verify(storagePoolVoMock).isManaged(); + Mockito.verify(storagePoolVoMock, Mockito.times(0)).getId(); + } + @Test public void executeManagedStorageChecksWhenTargetStoragePoolProvidedTestCurrentStoragePoolEqualsTargetPool() { Mockito.doReturn(true).when(storagePoolVoMock).isManaged(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index bc347b302aef..f00c2aca088e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -67,7 +67,7 @@ import javax.xml.transform.stream.StreamResult; @ResourceWrapper(handles = MigrateVolumeCommand.class) -public final class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { +public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper { private static final Logger LOGGER = Logger.getLogger(LibvirtMigrateVolumeCommandWrapper.class); @Override @@ -85,7 +85,7 @@ public Answer execute(final MigrateVolumeCommand command, final LibvirtComputing return answer; } - private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + protected MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { // Source Details VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); @@ -164,7 +164,7 @@ private MigrateVolumeAnswer migratePowerFlexVolume(final MigrateVolumeCommand co } } - private MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { + protected MigrateVolumeAnswer checkBlockJobStatus(MigrateVolumeCommand command, Domain dm, String diskLabel, String srcPath, String destPath, LibvirtComputingResource libvirtComputingResource, Connect conn, String srcSecretUUID) throws LibvirtException { int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if disk is found int waitTimeInSec = command.getWait(); while (waitTimeInSec > 0) { @@ -219,7 +219,7 @@ private String generateDestinationDiskLabel(String diskXml) throws ParserConfigu return diskLabel; } - private String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { + protected String generateDestinationDiskXML(Domain dm, String srcVolumeId, String diskFilePath, String destSecretUUID) throws LibvirtException, ParserConfigurationException, IOException, TransformerException, SAXException { final String domXml = dm.getXMLDesc(0); DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); @@ -270,7 +270,7 @@ private static void setAttrValue(String tag, String attr, String newValue, Eleme node.setAttribute(attr, newValue); } - private MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + protected MigrateVolumeAnswer migrateRegularVolume(final MigrateVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolManager = libvirtComputingResource.getStoragePoolMgr(); VolumeObjectTO srcVolumeObjectTO = (VolumeObjectTO)command.getSrcData(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java new file mode 100644 index 000000000000..c278144b4e1c --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapperTest.java @@ -0,0 +1,388 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.MigrateVolumeAnswer; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainBlockJobInfo; +import org.libvirt.DomainInfo; +import org.libvirt.LibvirtException; +import org.libvirt.TypedParameter; +import org.mockito.InjectMocks; +import org.mockito.Matchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtMigrateVolumeCommandWrapperTest { + + @Spy + @InjectMocks + private LibvirtMigrateVolumeCommandWrapper libvirtMigrateVolumeCommandWrapper; + + @Mock + MigrateVolumeCommand command; + + @Mock + LibvirtComputingResource libvirtComputingResource; + + @Mock + LibvirtUtilitiesHelper libvirtUtilitiesHelper; + + private String domxml = "\n" + + " i-2-27-VM\n" + + " 2d37fe1a-621a-4903-9ab5-5c9544c733f8\n" + + " Ubuntu 18.04 LTS\n" + + " 524288\n" + + " 524288\n" + + " 1\n" + + " \n" + + " 256\n" + + " \n" + + " \n" + + " /machine\n" + + " \n" + + " \n" + + " \n" + + " Apache Software Foundation\n" + + " CloudStack KVM Hypervisor\n" + + " 2d37fe1a-621a-4903-9ab5-5c9544c733f8\n" + + " \n" + + " \n" + + " \n" + + " hvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " qemu64\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " destroy\n" + + " restart\n" + + " destroy\n" + + " \n" + + " /usr/libexec/qemu-kvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 38a54bf719f24af6b070\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " 0ceeb7c643b447aba5ce\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "