From 22ec9904920ef646255977a70d7c5c3f2b809929 Mon Sep 17 00:00:00 2001 From: Mrunalini Kankariya Date: Thu, 20 Jul 2017 16:38:25 +0530 Subject: [PATCH 1/2] CLOUDSTACK-10054:Volume download times out in 3600 seconds --- .../cloud/storage/template/IsoProcessor.java | 5 ++ .../cloud/storage/template/OVAProcessor.java | 9 ++- .../com/cloud/storage/template/Processor.java | 2 + .../storage/template/QCOW2Processor.java | 5 ++ .../storage/template/RawImageProcessor.java | 5 ++ .../cloud/storage/template/TARProcessor.java | 5 ++ .../cloud/storage/template/VhdProcessor.java | 5 ++ .../cloud/storage/template/VmdkProcessor.java | 5 ++ .../TemplateOrVolumePostUploadCommand.java | 10 ++++ .../com/cloud/hypervisor/guru/VMwareGuru.java | 8 ++- .../vmware/manager/VmwareManager.java | 3 + .../vmware/manager/VmwareManagerImpl.java | 5 +- .../vmware/manager/VmwareStorageManager.java | 4 +- .../manager/VmwareStorageManagerImpl.java | 34 +++++++----- ...VmwareSecondaryStorageResourceHandler.java | 7 ++- .../VmwareStorageSubsystemCommandHandler.java | 12 ++-- .../cloud/storage/VolumeApiServiceImpl.java | 1 + .../resource/HttpUploadServerHandler.java | 6 +- .../resource/NfsSecondaryStorageResource.java | 7 ++- .../storage/template/DownloadManagerImpl.java | 6 +- .../storage/template/UploadEntity.java | 9 +++ .../java/com/cloud/utils/script/Script.java | 55 ++++++++++--------- 22 files changed, 146 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/IsoProcessor.java b/core/src/main/java/com/cloud/storage/template/IsoProcessor.java index 271818ccd918..7a65dde3f9ec 100644 --- a/core/src/main/java/com/cloud/storage/template/IsoProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/IsoProcessor.java @@ -36,6 +36,11 @@ public class IsoProcessor extends AdapterBase implements Processor { StorageLayer _storage; @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) { + return process(templatePath, format, templateName); + } + + @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) { if (format != null) { s_logger.debug("We don't handle conversion from " + format + " to ISO."); diff --git a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java index 08087bfb9c8f..f29efb46b52b 100644 --- a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java @@ -42,11 +42,15 @@ public class OVAProcessor extends AdapterBase implements Processor { private static final Logger s_logger = Logger.getLogger(OVAProcessor.class); - StorageLayer _storage; @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + return process(templatePath, format, templateName, 0); + } + + @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { if (format != null) { if (s_logger.isInfoEnabled()) { s_logger.info("We currently don't handle conversion from " + format + " to OVA."); @@ -66,8 +70,7 @@ public FormatInfo process(String templatePath, ImageFormat format, String templa s_logger.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName); String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension(); File templateFile = new File(templateFileFullPath); - - Script command = new Script("tar", 0, s_logger); + Script command = new Script("tar", processTimeout, s_logger); command.add("--no-same-owner"); command.add("--no-same-permissions"); command.add("-xf", templateFileFullPath); diff --git a/core/src/main/java/com/cloud/storage/template/Processor.java b/core/src/main/java/com/cloud/storage/template/Processor.java index ba57563e1b65..c8ee18109a14 100644 --- a/core/src/main/java/com/cloud/storage/template/Processor.java +++ b/core/src/main/java/com/cloud/storage/template/Processor.java @@ -44,6 +44,8 @@ public interface Processor extends Adapter { */ FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException; + FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException; + public static class FormatInfo { public ImageFormat format; public long size; diff --git a/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java b/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java index 642d8d3cb7b3..02780c1b710c 100644 --- a/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java +++ b/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java @@ -40,6 +40,11 @@ public class QCOW2Processor extends AdapterBase implements Processor { private StorageLayer _storage; + @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { + return process(templatePath, format, templateName); + } + @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { if (format != null) { diff --git a/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java b/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java index 5645a3199a73..3fdc253c7b6f 100644 --- a/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java @@ -45,6 +45,11 @@ public boolean configure(String name, Map params) throws Configu return true; } + @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { + return process(templatePath, format, templateName); + } + @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { if (format != null) { diff --git a/core/src/main/java/com/cloud/storage/template/TARProcessor.java b/core/src/main/java/com/cloud/storage/template/TARProcessor.java index dfd9a0adf25c..2daecda61c63 100644 --- a/core/src/main/java/com/cloud/storage/template/TARProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/TARProcessor.java @@ -33,6 +33,11 @@ public class TARProcessor extends AdapterBase implements Processor { private StorageLayer _storage; + @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) { + return process(templatePath, format, templateName); + } + @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) { if (format != null) { diff --git a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java index cb13d06687b8..7a73cd5b331e 100644 --- a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java @@ -56,6 +56,11 @@ public class VhdProcessor extends AdapterBase implements Processor { private byte[][] citrixCreatorApp = { {0x74, 0x61, 0x70, 0x00}, {0x43, 0x54, 0x58, 0x53}}; /*"tap ", and "CTXS"*/ private String vhdIdentifierCookie = "conectix"; + @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { + return process(templatePath, format, templateName); + } + @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { if (format != null) { diff --git a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java index eacabb939682..621da430ef55 100644 --- a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java @@ -42,6 +42,11 @@ public class VmdkProcessor extends AdapterBase implements Processor { StorageLayer _storage; + @Override + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { + return process(templatePath, format, templateName); + } + @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { if (format != null) { diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index 29fdd1911642..965d30320a8e 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -51,6 +51,8 @@ public class TemplateOrVolumePostUploadCommand { private String defaultMaxAccountSecondaryStorage; + private long processTimeout; + private long accountId; private Integer nfsVersion; @@ -206,4 +208,12 @@ public Integer getNfsVersion() { public void setNfsVersion(Integer nfsVersion) { this.nfsVersion = nfsVersion; } + + public void setProcessTimeout(long processTimeout) { + this.processTimeout = processTimeout; + } + + public long getProcessTimeout() { + return processTimeout; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index d8076fd5d68d..81dfc33bb883 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -434,13 +435,14 @@ public int compare(NicTO arg0, NicTO arg1) { @DB public Pair getCommandHostDelegation(long hostId, Command cmd) { boolean needDelegation = false; - if (cmd instanceof StorageSubSystemCommand) { Boolean fullCloneEnabled = VmwareFullClone.value(); StorageSubSystemCommand c = (StorageSubSystemCommand)cmd; c.setExecuteInSequence(fullCloneEnabled); } - + if (cmd instanceof DownloadCommand) { + cmd.setContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key(), String.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.value())); + } //NOTE: the hostid can be a hypervisor host, or a ssvm agent. For copycommand, if it's for volume upload, the hypervisor //type is empty, so we need to check the format of volume at first. if (cmd instanceof CopyCommand) { @@ -514,11 +516,11 @@ public Pair getCommandHostDelegation(long hostId, Command cmd) { cmd.setContextParam("execid", String.valueOf(execLog.getId())); cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId())); cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + cmd.setContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key(), String.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.value())); if (cmd instanceof BackupSnapshotCommand || cmd instanceof CreatePrivateTemplateFromVolumeCommand || cmd instanceof CreatePrivateTemplateFromSnapshotCommand || cmd instanceof CopyVolumeCommand || cmd instanceof CopyCommand || cmd instanceof CreateVolumeOVACommand || cmd instanceof PrepareOVAPackingCommand || cmd instanceof CreateVolumeFromSnapshotCommand) { - String workerName = _vmwareMgr.composeWorkerName(); long checkPointId = 1; // FIXME: Fix long checkPointId = _checkPointMgr.pushCheckPoint(new VmwareCleanupMaid(hostDetails.get("guid"), workerName)); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java index 4a4d2ea7a532..efdbc724fbde 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -42,6 +42,9 @@ public interface VmwareManager { static final ConfigKey s_vmwareSearchExcludeFolder = new ConfigKey("Advanced", String.class, "vmware.search.exclude.folders", null, "Comma seperated list of Datastore Folders to exclude from VMWare search", true, ConfigKey.Scope.Global); + static final ConfigKey s_vmwareOVAPackageTimeout = new ConfigKey(Integer.class, "vmware.package.ova.timeout", "Advanced", "3600", + "Vmware script timeout for ova packaging process", true, ConfigKey.Scope.Global, 1000); + String composeWorkerName(); String getSystemVMIsoFileNameOnDatastore(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index b7149ab2bf9e..3cf0c0000d99 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -136,6 +136,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private static final Logger s_logger = Logger.getLogger(VmwareManagerImpl.class); private static final long SECONDS_PER_MINUTE = 60; + private int _timeout; private String _instance; @@ -204,7 +205,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private int _additionalPortRangeSize; private int _routerExtraPublicNics = 2; private int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds - private String _rootDiskController = DiskControllerType.ide.toString(); private final String _dataDiskController = DiskControllerType.osdefault.toString(); @@ -229,9 +229,8 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder}; + return new ConfigKey[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout}; } - @Override public boolean configure(String name, Map params) throws ConfigurationException { s_logger.info("Configure VmwareManagerImpl, manager name: " + name); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java index f78f370da318..e7cd91946db3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java @@ -51,7 +51,7 @@ public interface VmwareStorageManager { boolean execute(VmwareHostService hostService, CreateEntityDownloadURLCommand cmd); - public void createOva(String path, String name); + public void createOva(String path, String name, int archiveTimeout); - public String createOvaForTemplate(TemplateObjectTO template); + public String createOvaForTemplate(TemplateObjectTO template, int archiveTimeout); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index a22410f4afeb..b6f207acf103 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -98,28 +98,32 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { @Override public boolean execute(VmwareHostService hostService, CreateEntityDownloadURLCommand cmd) { DataTO data = cmd.getData(); + int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()), + Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier()); if (data == null) { return false; } String newPath = null; if (data.getObjectType() == DataObjectType.VOLUME) { - newPath = createOvaForVolume((VolumeObjectTO)data); + newPath = createOvaForVolume((VolumeObjectTO)data, timeout); } else if (data.getObjectType() == DataObjectType.TEMPLATE) { - newPath = createOvaForTemplate((TemplateObjectTO)data); + newPath = createOvaForTemplate((TemplateObjectTO)data, timeout); } if (newPath != null) { cmd.setInstallPath(newPath); + return true; } - return true; + return false; + } @Override - public void createOva(String path, String name) { + public void createOva(String path, String name, int archiveTimeout) { Script commandSync = new Script(true, "sync", 0, s_logger); commandSync.execute(); - Script command = new Script(false, "tar", 0, s_logger); + Script command = new Script(false, "tar", archiveTimeout, s_logger); command.setWorkDir(path); command.add("-cf", name + ".ova"); command.add(name + ".ovf"); // OVF file should be the first file in OVA archive @@ -155,7 +159,7 @@ public void configure(Map params) { } @Override - public String createOvaForTemplate(TemplateObjectTO template) { + public String createOvaForTemplate(TemplateObjectTO template, int archiveTimeout) { DataStoreTO storeTO = template.getDataStore(); if (!(storeTO instanceof NfsTO)) { s_logger.debug("Can only handle NFS storage, while creating OVA from template"); @@ -173,7 +177,7 @@ public String createOvaForTemplate(TemplateObjectTO template) { s_logger.debug("OVA file found at: " + installFullPath); } else { if (new File(installFullPath + ".meta").exists()) { - createOVAFromMetafile(installFullPath + ".meta"); + createOVAFromMetafile(installFullPath + ".meta", archiveTimeout); } else { String msg = "Unable to find OVA or OVA MetaFile to prepare template."; s_logger.error(msg); @@ -190,7 +194,7 @@ public String createOvaForTemplate(TemplateObjectTO template) { //Fang: new command added; // Important! we need to sync file system before we can safely use tar to work around a linux kernal bug(or feature) - public String createOvaForVolume(VolumeObjectTO volume) { + public String createOvaForVolume(VolumeObjectTO volume, int archiveTimeout) { DataStoreTO storeTO = volume.getDataStore(); if (!(storeTO instanceof NfsTO)) { s_logger.debug("can only handle nfs storage, when create ova from volume"); @@ -215,15 +219,17 @@ public String createOvaForVolume(VolumeObjectTO volume) { } else { Script commandSync = new Script(true, "sync", 0, s_logger); commandSync.execute(); - - Script command = new Script(false, "tar", 0, s_logger); + Script command = new Script(false, "tar", archiveTimeout, s_logger); command.setWorkDir(installFullPath); command.add("-cf", volumeUuid + ".ova"); command.add(volumeUuid + ".ovf"); // OVF file should be the first file in OVA archive command.add(volumeUuid + "-disk0.vmdk"); - command.execute(); - return volumePath; + String result = command.execute(); + if (result != Script.ERR_TIMEOUT) { + return volumePath; + } + } } catch (Throwable e) { s_logger.info("Exception for createVolumeOVA"); @@ -1046,7 +1052,7 @@ private Pair copyVolumeFromSecStorage(VmwareHypervisorHost hyper // here we use a method to return the ovf and vmdk file names; Another way to do it: // create a new class, and like TemplateLocation.java and create templateOvfInfo.java to handle it; - private String createOVAFromMetafile(String metafileName) throws Exception { + private String createOVAFromMetafile(String metafileName, int archiveTimeout) throws Exception { File ova_metafile = new File(metafileName); Properties props = null; String ovaFileName = ""; @@ -1080,7 +1086,7 @@ private String createOVAFromMetafile(String metafileName) throws Exception { s_logger.info("ova: " + ovaFileName + ", ovf:" + ovfFileName + ", vmdk:" + disks[0] + "."); Script commandSync = new Script(true, "sync", 0, s_logger); commandSync.execute(); - Script command = new Script(false, "tar", 0, s_logger); + Script command = new Script(false, "tar", archiveTimeout, s_logger); command.setWorkDir(exportDir); // Fang: pass this in to the method? command.add("-cf", ovaFileName); command.add(ovfFileName); // OVF file should be the first file in OVA archive diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index d0d5964bfa92..ff00f55c8bcb 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -145,8 +145,11 @@ public Answer executeRequest(Command cmd) { } protected Answer execute(CreateEntityDownloadURLCommand cmd) { - _storageMgr.execute(this, cmd); - return _resource.defaultAction(cmd); + boolean success = _storageMgr.execute(this, cmd); + if (success) { + return _resource.defaultAction(cmd); + } + return new Answer(cmd, false, "Failed to download"); } private Answer execute(PrimaryStorageDownloadCommand cmd) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java index fc199722b2bb..02ae2fe50334 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java @@ -21,6 +21,8 @@ import java.io.File; import java.util.EnumMap; +import com.cloud.hypervisor.vmware.manager.VmwareManager; +import com.cloud.utils.NumbersUtil; import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; @@ -95,6 +97,8 @@ protected Answer execute(CopyCommand cmd) { DataTO destData = cmd.getDestTO(); DataStoreTO srcDataStore = srcData.getDataStore(); DataStoreTO destDataStore = destData.getDataStore(); + int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()), + Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier()); //if copied between s3 and nfs cache, go to resource boolean needDelegation = false; if (destDataStore instanceof NfsTO && destDataStore.getRole() == DataStoreRole.ImageCache) { @@ -112,11 +116,11 @@ protected Answer execute(CopyCommand cmd) { String path = vol.getPath(); int index = path.lastIndexOf(File.separator); String name = path.substring(index + 1); - storageManager.createOva(parentPath + File.separator + path, name); + storageManager.createOva(parentPath + File.separator + path, name, timeout); vol.setPath(path + File.separator + name + ".ova"); } else if (srcData.getObjectType() == DataObjectType.TEMPLATE) { // sync template from NFS cache to S3 in NFS migration to S3 case - storageManager.createOvaForTemplate((TemplateObjectTO)srcData); + storageManager.createOvaForTemplate((TemplateObjectTO)srcData, timeout); } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT) { // pack ova first // sync snapshot from NFS cache to S3 in NFS migration to S3 case @@ -126,7 +130,7 @@ protected Answer execute(CopyCommand cmd) { int index = path.lastIndexOf(File.separator); String name = path.substring(index + 1); String snapDir = path.substring(0, index); - storageManager.createOva(parentPath + File.separator + snapDir, name); + storageManager.createOva(parentPath + File.separator + snapDir, name, timeout); if (destData.getObjectType() == DataObjectType.TEMPLATE) { //create template from snapshot on src at first, then copy it to s3 TemplateObjectTO cacheTemplate = (TemplateObjectTO)destData; @@ -169,7 +173,7 @@ protected Answer execute(CopyCommand cmd) { int index = path.lastIndexOf(File.separator); String name = path.substring(index + 1); String dir = path.substring(0, index); - storageManager.createOva(parentPath + File.separator + dir, name); + storageManager.createOva(parentPath + File.separator + dir, name, timeout); newSnapshot.setPath(newSnapshot.getPath() + ".ova"); newSnapshot.setDataStore(cmd.getCacheTO().getDataStore()); CopyCommand newCmd = new CopyCommand(newSnapshot, destData, cmd.getWait(), cmd.executeInSequence()); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index bea9b4ad5bfb..69dfd1d86f35 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -367,6 +367,7 @@ public GetUploadParamsResponse doInTransaction(TransactionStatus status) throws dataObject.getDataStore().getRole().toString()); command.setLocalPath(volumeStore.getLocalDownloadPath()); //using the existing max upload size configuration + command.setProcessTimeout(NumbersUtil.parseLong(_configDao.getValue("vmware.package.ova.timeout"), 3600)); command.setMaxUploadSize(_configDao.getValue(Config.MaxUploadVolumeSize.key())); command.setDefaultMaxAccountSecondaryStorage(_configDao.getValue(Config.DefaultMaxAccountSecondaryStorage.key())); command.setAccountId(vol.getAccountId()); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java index 7a22b20ffdb4..ea4e75d89430 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java @@ -87,6 +87,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler> uriAttributes = decoderQuery.parameters(); uuid = uriAttributes.get("uuid").get(0); logger.info("URI: uuid=" + uuid); - UploadEntity uploadEntity = null; try { // Validate the request here @@ -175,6 +176,7 @@ public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Excep } //set the base directory to download the file DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix(); + this.processTimeout = uploadEntity.getProcessTimeout(); logger.info("base directory: " + DiskFileUpload.baseDirectory); try { //initialize the decoder @@ -243,7 +245,7 @@ private HttpResponseStatus readFileUploadData() throws IOException { storageResource.updateStateMapWithError(uuid, errorString); return HttpResponseStatus.BAD_REQUEST; } - String status = storageResource.postUpload(uuid, fileUpload.getFile().getName()); + String status = storageResource.postUpload(uuid, fileUpload.getFile().getName(), processTimeout); if (status != null) { responseContent.append(status); storageResource.updateStateMapWithError(uuid, status); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 93d8c271df33..1f81f6aa841b 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -251,6 +251,7 @@ public void setTimeout(int timeout) { protected String createTemplateFromSnapshotXenScript; private HashMap uploadEntityStateMap = new HashMap(); private String _ssvmPSK = null; + private long processTimeout; public void setParentPath(String path) { _parent = path; @@ -3345,6 +3346,7 @@ public UploadEntity createUploadEntity(String uuid, String metadata, long conten throw new InvalidParameterValueException(errorMessage); } else { uuid = cmd.getEntityUUID(); + processTimeout = cmd.getProcessTimeout(); if (isOneTimePostUrlUsed(cmd)) { uploadEntity = uploadEntityStateMap.get(uuid); StringBuilder errorMessage = new StringBuilder("The one time post url is already used"); @@ -3366,6 +3368,7 @@ public UploadEntity createUploadEntity(String uuid, String metadata, long conten uploadEntity = new UploadEntity(uuid, cmd.getEntityId(), UploadEntity.Status.IN_PROGRESS, cmd.getName(), absolutePath); uploadEntity.setMetaDataPopulated(true); uploadEntity.setResourceType(UploadEntity.ResourceType.valueOf(cmd.getType())); + uploadEntity.setProcessTimeout(processTimeout); uploadEntity.setFormat(Storage.ImageFormat.valueOf(cmd.getImageFormat())); //relative path with out ssvm mount info. uploadEntity.setTemplatePath(absolutePath); @@ -3447,7 +3450,7 @@ private int getSizeInGB(long sizeInBytes) { return (int)Math.ceil(sizeInBytes * 1.0d / (1024 * 1024 * 1024)); } - public String postUpload(String uuid, String filename) { + public String postUpload(String uuid, String filename, long processTimeout) { UploadEntity uploadEntity = uploadEntityStateMap.get(uuid); int installTimeoutPerGig = 180 * 60 * 1000; @@ -3552,7 +3555,7 @@ public String postUpload(String uuid, String filename) { for (Processor processor : processors.values()) { FormatInfo info = null; try { - info = processor.process(resourcePath, null, templateName); + info = processor.process(resourcePath, null, templateName, processTimeout * 1000); } catch (InternalErrorException e) { s_logger.error("Template process exception ", e); return e.toString(); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index 5fa13a34cfc3..2ace37f675a5 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -90,7 +90,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private String _name; StorageLayer _storage; public Map _processors; - + private long _processTimeout; private Integer _nfsVersion; public class Completion implements DownloadCompleteCallback { @@ -459,7 +459,7 @@ private String postLocalDownload(String jobId) { FormatInfo info = null; try { - info = processor.process(resourcePath, null, templateName); + info = processor.process(resourcePath, null, templateName, this._processTimeout); } catch (InternalErrorException e) { s_logger.error("Template process exception ", e); return e.toString(); @@ -677,6 +677,8 @@ public com.cloud.storage.VMTemplateHostVO.Status getDownloadStatus2(String jobId @Override public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) { + int timeout = NumbersUtil.parseInt(cmd.getContextParam("vmware.package.ova.timeout"), 3600000); + this._processTimeout = timeout; ResourceType resourceType = cmd.getResourceType(); if (cmd instanceof DownloadProgressCommand) { return handleDownloadProgressCmd(resource, (DownloadProgressCommand)cmd); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadEntity.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadEntity.java index d851143b200f..031a163997e5 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadEntity.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadEntity.java @@ -34,6 +34,7 @@ public class UploadEntity { private int maxSizeInGB; private String description; private long contentLength; + private long processTimeout; public static enum ResourceType { VOLUME, TEMPLATE @@ -60,6 +61,14 @@ public UploadEntity(String uuid, long entityId, Status status, String filename, this.entityId=entityId; } + public void setProcessTimeout(long processTimeout) { + this.processTimeout = processTimeout; + } + + public long getProcessTimeout() { + return processTimeout; + } + public UploadEntity(){ } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index 01f18bda2d2e..7087395b050f 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -37,7 +37,6 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; import org.joda.time.Duration; @@ -203,7 +202,7 @@ public String execute(OutputInterpreter interpreter) { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { - _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.defaultKeystoreFile)[0]); + _logger.debug("Executing: " + buildCommandLine(command)); } try { @@ -233,18 +232,23 @@ public String execute(OutputInterpreter interpreter) { } while (true) { + _logger.debug("Executing while with timeout : " + _timeout); try { - if (_process.waitFor() == 0) { - _logger.debug("Execution is successful."); - if (interpreter != null) { - return interpreter.drain() ? task.getResult() : interpreter.interpret(ir); - } else { - // null return exitValue apparently - return String.valueOf(_process.exitValue()); + //process execution completed within timeout period + if (_process.waitFor(_timeout, TimeUnit.MILLISECONDS)) { + //process completed successfully + if (_process.exitValue() == 0) { + _logger.debug("Execution is successful."); + if (interpreter != null) { + return interpreter.drain() ? task.getResult() : interpreter.interpret(ir); + } else { + // null return exitValue apparently + return String.valueOf(_process.exitValue()); + } + } else { //process failed + break; } - } else { - break; - } + } //timeout } catch (InterruptedException e) { if (!_isTimeOut) { /* @@ -254,24 +258,25 @@ public String execute(OutputInterpreter interpreter) { _logger.debug("We are interrupted but it's not a timeout, just continue"); continue; } - - TimedOutLogger log = new TimedOutLogger(_process); - Task timedoutTask = new Task(log, ir); - - timedoutTask.run(); - if (!_passwordCommand) { - _logger.warn("Timed out: " + buildCommandLine(command) + ". Output is: " + timedoutTask.getResult()); - } else { - _logger.warn("Timed out: " + buildCommandLine(command)); - } - - return ERR_TIMEOUT; } finally { if (future != null) { future.cancel(false); } Thread.interrupted(); } + + //timeout without completing the process + TimedOutLogger log = new TimedOutLogger(_process); + Task timedoutTask = new Task(log, ir); + + timedoutTask.run(); + if (!_passwordCommand) { + _logger.warn("Timed out: " + buildCommandLine(command) + ". Output is: " + timedoutTask.getResult()); + } else { + _logger.warn("Timed out: " + buildCommandLine(command)); + } + + return ERR_TIMEOUT; } _logger.debug("Exit value is " + _process.exitValue()); @@ -300,7 +305,7 @@ public String execute(OutputInterpreter interpreter) { IOUtils.closeQuietly(_process.getErrorStream()); IOUtils.closeQuietly(_process.getOutputStream()); IOUtils.closeQuietly(_process.getInputStream()); - _process.destroy(); + _process.destroyForcibly(); } } } From 8a2dcaf9c4a98dcff8a1b6a6a1b6d14362e309ed Mon Sep 17 00:00:00 2001 From: mrunalinikankariya Date: Tue, 13 Feb 2018 18:27:05 +0530 Subject: [PATCH 2/2] CLOUDSTACK-10054:Volume download times out in 3600 seconds --- .../main/java/com/cloud/storage/template/IsoProcessor.java | 6 +++--- .../java/com/cloud/storage/template/QCOW2Processor.java | 6 +++--- .../java/com/cloud/storage/template/RawImageProcessor.java | 6 +++--- .../main/java/com/cloud/storage/template/TARProcessor.java | 6 +++--- .../main/java/com/cloud/storage/template/VhdProcessor.java | 6 +++--- .../main/java/com/cloud/storage/template/VmdkProcessor.java | 6 +++--- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/com/cloud/storage/template/IsoProcessor.java b/core/src/main/java/com/cloud/storage/template/IsoProcessor.java index 7a65dde3f9ec..4cd2f1a2a020 100644 --- a/core/src/main/java/com/cloud/storage/template/IsoProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/IsoProcessor.java @@ -36,12 +36,12 @@ public class IsoProcessor extends AdapterBase implements Processor { StorageLayer _storage; @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) { - return process(templatePath, format, templateName); + public FormatInfo process(String templatePath, ImageFormat format, String templateName) { + return process(templatePath, format, templateName, 0); } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName) { + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) { if (format != null) { s_logger.debug("We don't handle conversion from " + format + " to ISO."); return null; diff --git a/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java b/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java index 02780c1b710c..56ae078dc519 100644 --- a/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java +++ b/core/src/main/java/com/cloud/storage/template/QCOW2Processor.java @@ -41,12 +41,12 @@ public class QCOW2Processor extends AdapterBase implements Processor { private StorageLayer _storage; @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { - return process(templatePath, format, templateName); + public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + return process(templatePath, format, templateName, 0); } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { if (format != null) { s_logger.debug("We currently don't handle conversion from " + format + " to QCOW2."); return null; diff --git a/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java b/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java index 3fdc253c7b6f..5fbc626f271e 100644 --- a/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/RawImageProcessor.java @@ -46,12 +46,12 @@ public boolean configure(String name, Map params) throws Configu } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { - return process(templatePath, format, templateName); + public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + return process(templatePath, format, templateName, 0); } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { if (format != null) { s_logger.debug("We currently don't handle conversion from " + format + " to raw image."); return null; diff --git a/core/src/main/java/com/cloud/storage/template/TARProcessor.java b/core/src/main/java/com/cloud/storage/template/TARProcessor.java index 2daecda61c63..51aeb234c50d 100644 --- a/core/src/main/java/com/cloud/storage/template/TARProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/TARProcessor.java @@ -34,12 +34,12 @@ public class TARProcessor extends AdapterBase implements Processor { private StorageLayer _storage; @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) { - return process(templatePath, format, templateName); + public FormatInfo process(String templatePath, ImageFormat format, String templateName) { + return process(templatePath, format, templateName, 0); } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName) { + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) { if (format != null) { s_logger.debug("We currently don't handle conversion from " + format + " to TAR."); return null; diff --git a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java index 7a73cd5b331e..baea7bf0db51 100644 --- a/core/src/main/java/com/cloud/storage/template/VhdProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VhdProcessor.java @@ -57,12 +57,12 @@ public class VhdProcessor extends AdapterBase implements Processor { private String vhdIdentifierCookie = "conectix"; @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { - return process(templatePath, format, templateName); + public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + return process(templatePath, format, templateName, 0); } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { if (format != null) { s_logger.debug("We currently don't handle conversion from " + format + " to VHD."); return null; diff --git a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java index 621da430ef55..ee50b2718b7d 100644 --- a/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/VmdkProcessor.java @@ -43,12 +43,12 @@ public class VmdkProcessor extends AdapterBase implements Processor { StorageLayer _storage; @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { - return process(templatePath, format, templateName); + public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + return process(templatePath, format, templateName, 0); } @Override - public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException { + public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { if (format != null) { if (s_logger.isInfoEnabled()) { s_logger.info("We currently don't handle conversion from " + format + " to VMDK.");