CLOUDSTACK-7982: KVM live migration with local storage

api/src/main/java/com/cloud/vm/DiskProfile.java

@@ -72,6 +72,7 @@ public DiskProfile(Volume vol, DiskOffering offering, HypervisorType hyperType)
             offering.isCustomized(),
             null);
         this.hyperType = hyperType;
+        this.path = vol.getPath();
     }
 
     public DiskProfile(DiskProfile dp) {

core/src/main/java/com/cloud/agent/api/CancelMigrationAnswer.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+public class CancelMigrationAnswer extends Answer {
+
+    public CancelMigrationAnswer(Command cmd, Exception e) {
+        super(cmd, e);
+    }
+    public CancelMigrationAnswer(Command cmd, boolean success, String details) {
+        super(cmd, success, details);
+    }
+}

core/src/main/java/com/cloud/agent/api/CancelMigrationCommand.java

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+public class CancelMigrationCommand extends Command {
+    String vmName;
+    boolean executeInSequence = false;
+
+    public CancelMigrationCommand(String vmName) {
+        this.vmName = vmName;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return executeInSequence;
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateWithStorageAnswer.java

@@ -25,19 +25,26 @@
 
 public class MigrateWithStorageAnswer extends Answer {
 
-    List<VolumeObjectTO> volumeTos;
+    final List<VolumeObjectTO> volumeTos;
+    boolean aborted = false;
 
-    public MigrateWithStorageAnswer(MigrateWithStorageCommand cmd, Exception ex) {
-        super(cmd, ex);
+    public MigrateWithStorageAnswer(MigrateWithStorageCommand cmd, boolean result, boolean aborted, Exception ex) {
+        super(cmd, result, ex.toString());
+        this.aborted = aborted;
         volumeTos = null;
     }
 
-    public MigrateWithStorageAnswer(MigrateWithStorageCommand cmd, List<VolumeObjectTO> volumeTos) {
-        super(cmd, true, null);
+    public MigrateWithStorageAnswer(MigrateWithStorageCommand cmd, List<VolumeObjectTO> volumeTos, boolean aborted, String details) {
+        super(cmd, !aborted, details);
         this.volumeTos = volumeTos;
+        this.aborted = aborted;
     }
 
     public List<VolumeObjectTO> getVolumeTos() {
         return volumeTos;
     }
+
+    public boolean isAborted() {
+        return aborted;
+    }
 }

engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java

@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.exception.VirtualMachineMigrationException;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.to.NicTO;
@@ -118,7 +119,7 @@ void orchestrateStart(String vmUuid, Map<VirtualMachineProfile.Param, Object> pa
 
     void migrate(String vmUuid, long srcHostId, DeployDestination dest) throws ResourceUnavailableException, ConcurrentOperationException;
 
-    void migrateWithStorage(String vmUuid, long srcId, long destId, Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException;
+    void migrateWithStorage(String vmUuid, long srcId, long destId, Map<Long, Long> volumeToPool) throws ResourceUnavailableException, ConcurrentOperationException, VirtualMachineMigrationException;
 
     void reboot(String vmUuid, Map<VirtualMachineProfile.Param, Object> params) throws InsufficientCapacityException, ResourceUnavailableException;
 

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java

@@ -21,6 +21,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import com.cloud.exception.VirtualMachineMigrationException;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
@@ -105,12 +106,14 @@ DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Lon
 
     void revokeAccess(long vmId, long hostId);
 
-    void migrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool);
+    void liveMigrateVolumes(VirtualMachine vm, VirtualMachineTO vmTo, Host srcHost, Host destHost, Map<Volume, StoragePool> volumeToPool) throws VirtualMachineMigrationException;
 
     boolean storageMigration(VirtualMachineProfile vm, StoragePool destPool) throws StorageUnavailableException;
 
     void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest);
 
+    void confirmMigration(VirtualMachineProfile vm, long srcHostId, long destHostId, boolean migrationSuccess);
+
     void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException;
 
     boolean canVmRestartOnAnotherServer(long vmId);

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/CopyCommandResult.java

@@ -39,4 +39,10 @@ public String getPath() {
     public Answer getAnswer() {
         return this.answer;
     }
+
+    @Override
+    public String toString() {
+        String sup = super.toString();
+        return sup.substring(0, sup.length()-1) + ", path: " + (path == null ? "<null>" : path) + ", answer: " + (answer == null ? "<null>" : answer) + "}";
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java

@@ -109,4 +109,6 @@ AsyncCallFuture<VolumeApiResult> createVolumeFromTemplateAsync(VolumeInfo volume
 
     VolumeInfo updateHypervisorSnapshotReserveForVolume(DiskOffering diskOffering, long volumeId, HypervisorType hyperType);
 
+    boolean deleteVolumeOnDataStore(DataStore store, long volumeId);
+
 }

engine/api/src/main/java/org/apache/cloudstack/storage/command/CommandResult.java

@@ -20,10 +20,12 @@
 
 public class CommandResult {
     private boolean success;
+    private boolean aborted;
     private String result;
 
     public CommandResult() {
         this.success = true;
+        this.aborted = false;
         this.result = "";
     }
 
@@ -39,6 +41,14 @@ public void setSuccess(boolean success) {
         this.success = success;
     }
 
+    public boolean isAborted() {
+        return aborted;
+    }
+
+    public void setAborted(boolean aborted) {
+        this.aborted = aborted;
+    }
+
     public String getResult() {
         return this.result;
     }
@@ -49,4 +59,9 @@ public void setResult(String result) {
             this.success = false;
         }
     }
+
+    @Override
+    public String toString() {
+        return "CommandResult={success: " + success + ", aborted: " + aborted + ", result: " + (result == null ? "<null>" : result) + "}";
+    }
 }

engine/orchestration/pom.xml

@@ -68,6 +68,7 @@
       <artifactId>cloud-server</artifactId>
       <version>${project.version}</version>
     </dependency>
+
   </dependencies>
   <build>
     <plugins>

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -74,6 +74,7 @@
 import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
 
+
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
 import com.cloud.agent.api.AgentControlAnswer;
@@ -147,6 +148,7 @@
 import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.exception.StorageUnavailableException;
+import com.cloud.exception.VirtualMachineMigrationException;
 import com.cloud.gpu.dao.VGPUTypesDao;
 import com.cloud.ha.HighAvailabilityManager;
 import com.cloud.ha.HighAvailabilityManager.WorkType;
@@ -901,7 +903,7 @@ public void advanceStart(final String vmUuid, final Map<VirtualMachineProfile.Pa
             } catch (final InterruptedException e) {
                 throw new RuntimeException("Operation is interrupted", e);
             } catch (final java.util.concurrent.ExecutionException e) {
-                throw new RuntimeException("Execution excetion", e);
+                throw new RuntimeException("Execution exception", e);
             }
 
             final Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob());
@@ -2437,7 +2439,7 @@ private <T extends VMInstanceVO> void moveVmOutofMigratingStateOnSuccess(final T
 
     @Override
     public void migrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool)
-            throws ResourceUnavailableException, ConcurrentOperationException {
+            throws ResourceUnavailableException, ConcurrentOperationException, VirtualMachineMigrationException {
 
         final AsyncJobExecutionContext jobContext = AsyncJobExecutionContext.getCurrentExecutionContext();
         if (jobContext.isJobDispatchedBy(VmWorkConstants.VM_WORK_JOB_DISPATCHER)) {
@@ -2481,7 +2483,7 @@ public void migrateWithStorage(final String vmUuid, final long srcHostId, final
     }
 
     private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHostId, final long destHostId, final Map<Long, Long> volumeToPool) throws ResourceUnavailableException,
-    ConcurrentOperationException {
+    ConcurrentOperationException, VirtualMachineMigrationException {
 
         final VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
 
@@ -2493,6 +2495,10 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
         final HostPodVO pod = _podDao.findById(destHost.getPodId());
         final Cluster cluster = _clusterDao.findById(destHost.getClusterId());
         final DeployDestination destination = new DeployDestination(dc, pod, cluster, destHost);
+        final VirtualMachineProfile vmSrc = new VirtualMachineProfileImpl(vm);
+        for (NicProfile nic : _networkMgr.getNicProfiles(vm)) {
+            vmSrc.addNic(nic);
+        }
 
         // Create a map of which volume should go in which storage pool.
         final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
@@ -2573,47 +2579,50 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo
             }
 
             // Migrate the vm and its volume.
-            volumeMgr.migrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap);
-
-            // Put the vm back to running state.
-            moveVmOutofMigratingStateOnSuccess(vm, destHost.getId(), work);
+            volumeMgr.liveMigrateVolumes(vm, to, srcHost, destHost, volumeToPoolMap);
+            migrated = true;
 
             try {
-                if (!checkVmOnHost(vm, destHostId)) {
+                if (migrated && !checkVmOnHost(vm, destHostId)) {
                     s_logger.error("Vm not found on destination host. Unable to complete migration for " + vm);
-                    try {
-                        _agentMgr.send(srcHostId, new Commands(cleanup(vm.getInstanceName())), null);
-                    } catch (final AgentUnavailableException e) {
-                        s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId);
-                    }
-                    cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true);
-                    throw new CloudRuntimeException("VM not found on desintation host. Unable to complete migration for " + vm);
+                    migrated = false;
                 }
             } catch (final OperationTimedoutException e) {
                 s_logger.warn("Error while checking the vm " + vm + " is on host " + destHost, e);
             }
 
-            migrated = true;
         } finally {
+
             if (!migrated) {
-                s_logger.info("Migration was unsuccessful.  Cleaning up: " + vm);
+                s_logger.info("Migration was unsuccessful for " + vm);
                 _alertMgr.sendAlert(alertType, srcHost.getDataCenterId(), srcHost.getPodId(),
                         "Unable to migrate vm " + vm.getInstanceName() + " from host " + srcHost.getName() + " in zone " + dc.getName() + " and pod " + dc.getName(),
-                        "Migrate Command failed.  Please check logs.");
+                        "Migrate Command failed. Please check logs.");
+                vm.setHostId(srcHostId);
+                _vmDao.update(vm.getId(), vm);
                 try {
-                    _agentMgr.send(destHostId, new Commands(cleanup(vm.getInstanceName())), null);
-                    vm.setPodIdToDeployIn(srcHost.getPodId());
+                    _networkMgr.rollbackNicForMigration(vmSrc, profile);
                     stateTransitTo(vm, Event.OperationFailed, srcHostId);
-                } catch (final AgentUnavailableException e) {
-                    s_logger.warn("Looks like the destination Host is unavailable for cleanup.", e);
-                } catch (final NoTransitionException e) {
+                } catch (NoTransitionException e) {
                     s_logger.error("Error while transitioning vm from migrating to running state.", e);
                 }
+            } else {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Migration was successful, committing it.");
+                }
+                // Put the vm back to running state.
+                moveVmOutofMigratingStateOnSuccess(vm, destHost.getId(), work);
             }
 
+            volumeMgr.confirmMigration(profile, srcHostId, destHostId, migrated);
+
             work.setStep(Step.Done);
             _workDao.update(work.getId(), work);
         }
+
+        if (!migrated) {
+            throw new VirtualMachineMigrationException("Migration failed");
+        }
     }
 
     @Override