From c09e5a81cb03109386f72c5ab681a9ffb493965d Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 12 May 2017 02:41:24 +0530 Subject: [PATCH] CLOUDSTACK-8612 [VMware] Make vCenter session timeout configurable for volume snapshot --- .../framework/jobs/dao/SyncQueueItemDao.java | 2 +- .../jobs/dao/SyncQueueItemDaoImpl.java | 48 ++++++++++++++----- .../jobs/impl/AsyncJobManagerImpl.java | 7 ++- .../framework/jobs/impl/SyncQueueManager.java | 2 +- .../jobs/impl/SyncQueueManagerImpl.java | 4 +- .../com/cloud/hypervisor/guru/VMwareGuru.java | 14 +++++- .../vmware/VmwareServerDiscoverer.java | 3 +- .../vmware/manager/VmwareManager.java | 3 ++ .../vmware/manager/VmwareManagerImpl.java | 15 +++++- .../vmware/resource/VmwareContextFactory.java | 16 +++---- .../vmware/resource/VmwareResource.java | 19 ++++++-- .../VmwareSecondaryStorageContextFactory.java | 18 +++---- ...VmwareSecondaryStorageResourceHandler.java | 35 +++++++++----- .../src/com/cloud/configuration/Config.java | 1 + setup/db/db/schema-4930to41000.sql | 5 +- .../vmware/util/VmwareContextPool.java | 9 ++-- .../vmware/mo/TestVmwareContextFactory.java | 12 ++--- .../vmware/util/VmwareContextPoolTest.java | 11 +++-- 18 files changed, 152 insertions(+), 72 deletions(-) diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java index 7b6eed75a008..798de4da4402 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java @@ -30,7 +30,7 @@ public interface SyncQueueItemDao extends GenericDao { public List getActiveQueueItems(Long msid, boolean exclusive); - public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive); public Long getQueueItemIdByContentIdAndType(long contentId, String contentType); } diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java index 29c3f1b289fb..09431b7c8afd 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java @@ -141,23 +141,45 @@ public List getActiveQueueItems(Long msid, boolean exclusive) { } @Override - public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive) { Date cutTime = DateUtil.currentGMTTime(); + List l = new ArrayList(); - SearchBuilder sbItem = createSearchBuilder(); - sbItem.and("lastProcessMsid", sbItem.entity().getLastProcessMsid(), SearchCriteria.Op.NNULL); - sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessNumber(), SearchCriteria.Op.NNULL); - sbItem.and("lastProcessTime", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.NNULL); - sbItem.and("lastProcessTime2", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.LT); - - sbItem.done(); + Date date1 = new Date(cutTime.getTime() - thresholdMs); + String dateString1 = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), date1); - SearchCriteria sc = sbItem.create(); - sc.setParameters("lastProcessTime2", new Date(cutTime.getTime() - thresholdMs)); + Date date2 = new Date(cutTime.getTime() - snapshotThresholdMs); + String dateString2 = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), date2); - if(exclusive) - return lockRows(sc, null, true); - return listBy(sc, null); + String sql = "SELECT s.id, s.queue_id, s.content_type, s.content_id, s.queue_proc_msid, s.queue_proc_number, s.queue_proc_time, s.created FROM sync_queue_item AS s " + + "JOIN async_job as a ON s.content_id = a.id WHERE s.queue_proc_msid IS NOT NULL AND s.queue_proc_number IS NOT NULL AND s.queue_proc_time IS NOT NULL" + + " AND ((a.job_cmd NOT LIKE ? AND s.queue_proc_time < ? ) OR (a.job_cmd LIKE ? AND s.queue_proc_time < ?))"; + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) { + pstmt.setString(1, "%" + jobCmd + "%"); + pstmt.setString(2, dateString1); + pstmt.setString(3, "%" + jobCmd + "%"); + pstmt.setString(4, dateString2); + try (ResultSet rs = pstmt.executeQuery()) { + while (rs.next()) { + SyncQueueItemVO item = new SyncQueueItemVO(); + item.setId(rs.getLong(1)); + item.setQueueId(rs.getLong(2)); + item.setContentType(rs.getString(3)); + item.setContentId(rs.getLong(4)); + item.setLastProcessMsid(rs.getLong(5)); + item.setLastProcessNumber(rs.getLong(6)); + item.setLastProcessTime(rs.getDate(7)); + item.setCreated(rs.getDate(8)); + l.add(item); + } + } + } catch (SQLException e) { + s_logger.error("Unexpected sql excetpion, ", e); + } catch (Throwable e) { + s_logger.error("Unexpected excetpion, ", e); + } + return l; } @Override diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 9b6aa7ccc613..d57b20aa9833 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -94,6 +94,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private static final ConfigKey VmJobLockTimeout = new ConfigKey("Advanced", Integer.class, "vm.job.lock.timeout", "1800", "Time in seconds to wait in acquiring lock to submit a vm worker job", false); + private static final ConfigKey VolumeSnapshotJobCancelThreshold = new ConfigKey("Advanced", Long.class, "volume.snapshot.job.cancel.threshold", "361", + "Time (in minutes) for volume snapshot async-job to be forcefully cancelled if it has been in process for long", true, ConfigKey.Scope.Global); private static final Logger s_logger = Logger.getLogger(AsyncJobManagerImpl.class); @@ -137,7 +139,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout}; + return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout, VolumeSnapshotJobCancelThreshold}; } @Override @@ -813,7 +815,8 @@ public void reallyRun() { s_logger.info("Begin cleanup expired async-jobs"); // forcefully cancel blocking queue items if they've been staying there for too long - List blockItems = _queueMgr.getBlockedQueueItems(JobCancelThresholdMinutes.value() * 60000, false); + List blockItems = _queueMgr.getBlockedQueueItems(JobCancelThresholdMinutes.value() * 60000, VolumeSnapshotJobCancelThreshold.value() * 60000, + "VmWorkTakeVolumeSnapshot", false); if (blockItems != null && blockItems.size() > 0) { for (SyncQueueItemVO item : blockItems) { try { diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java index 32d84647a2d4..f5894696b842 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java @@ -33,7 +33,7 @@ public interface SyncQueueManager extends Manager { public List getActiveQueueItems(Long msid, boolean exclusive); - public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive); void purgeAsyncJobQueueItemId(long asyncJobId); diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java index 2f97991e3e31..dac70372d0b2 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java @@ -238,8 +238,8 @@ public List getActiveQueueItems(Long msid, boolean exclusive) { } @Override - public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { - return _syncQueueItemDao.getBlockedQueueItems(thresholdMs, exclusive); + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive) { + return _syncQueueItemDao.getBlockedQueueItems(thresholdMs, snapshotThresholdMs, jobCmd, exclusive); } private boolean queueReadyToProcess(SyncQueueVO queueVO) { diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java index 64ca4067ceee..1195438dcbed 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java @@ -492,7 +492,19 @@ public Pair getCommandHostDelegation(long hostId, Command cmd) { _cmdExecLogDao.persist(execLog); cmd.setContextParam("execid", String.valueOf(execLog.getId())); cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId())); - cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + if (cmd instanceof CopyCommand) { // Snapshot backup + CopyCommand cpyCommand = (CopyCommand)cmd; + DataTO srcData = cpyCommand.getSrcTO(); + DataTO destData = cpyCommand.getDestTO(); + if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.SNAPSHOT + && srcData.getDataStore().getRole() == DataStoreRole.Primary) { + cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getSnapshotBackupSessionTimeout())); + } else { + cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + } + } else { + cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + } if (cmd instanceof BackupSnapshotCommand || cmd instanceof CreatePrivateTemplateFromVolumeCommand || cmd instanceof CreatePrivateTemplateFromSnapshotCommand || cmd instanceof CopyVolumeCommand || cmd instanceof CopyCommand || diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java index 4b2f830a3ee7..9b6ef3a3dbc9 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java @@ -311,7 +311,8 @@ public VmwareServerDiscoverer() { VmwareContext context = null; try { - context = VmwareContextFactory.create(url.getHost(), username, password); + int sessionTimeout = _vmwareMgr.getVcenterSessionTimeout(); + context = VmwareContextFactory.create(url.getHost(), username, password, sessionTimeout); if (privateTrafficLabel != null) context.registerStockObject("privateTrafficLabel", privateTrafficLabel); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java index 12c48fee026d..54469c33f4df 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -87,5 +87,8 @@ public interface VmwareManager { boolean isLegacyZone(long dcId); public String getDataDiskController(); + boolean hasNexusVSM(Long clusterId); + + public int getSnapshotBackupSessionTimeout(); } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index b0f87af648dc..1913ff06222a 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -191,6 +191,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private int _additionalPortRangeSize; private int _routerExtraPublicNics = 2; private int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds + private int _snapshotBackupSessionTimeout = 1200000; // Timeout in milliseconds private String _rootDiskController = DiskControllerType.ide.toString(); @@ -301,6 +302,9 @@ public boolean configure(String name, Map params) throws Configu _vCenterSessionTimeout = NumbersUtil.parseInt(_configDao.getValue(Config.VmwareVcenterSessionTimeout.key()), 1200) * 1000; s_logger.info("VmwareManagerImpl config - vmware.vcenter.session.timeout: " + _vCenterSessionTimeout); + _snapshotBackupSessionTimeout = NumbersUtil.parseInt(_configDao.getValue(Config.VmwareSnapshotBackupSessionTimeout.key()), 1200) * 1000; + s_logger.info("VmwareManagerImpl config - vmware.snapshot.backup.session.timeout: " + _snapshotBackupSessionTimeout); + _recycleHungWorker = _configDao.getValue(Config.VmwareRecycleHungWorker.key()); if (_recycleHungWorker == null || _recycleHungWorker.isEmpty()) { _recycleHungWorker = "false"; @@ -510,6 +514,8 @@ public void setupResourceStartupParams(Map params) { params.put("vmware.data.disk.controller", _dataDiskController); params.put("vmware.recycle.hung.wokervm", _recycleHungWorker); params.put("ports.per.dvportgroup", _portsPerDvPortGroup); + params.put("vmware.vcenter.session.timeout", _vCenterSessionTimeout); + params.put("vmware.snapshot.backup.session.timeout", _snapshotBackupSessionTimeout); } @Override @@ -994,6 +1000,11 @@ public int getVcenterSessionTimeout() { return _vCenterSessionTimeout; } + @Override + public int getSnapshotBackupSessionTimeout() { + return _snapshotBackupSessionTimeout; + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -1074,7 +1085,7 @@ public VmwareDatacenterVO addVmwareDatacenter(AddVmwareDcCmd cmd) throws Resourc String guid; ManagedObjectReference dcMor; try { - context = VmwareContextFactory.create(vCenterHost, userName, password); + context = VmwareContextFactory.create(vCenterHost, userName, password, _vCenterSessionTimeout); // Check if DC exists on vCenter dcMo = new DatacenterMO(context, vmwareDcName); @@ -1171,7 +1182,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { // Construct context VmwareContext context = null; try { - context = VmwareContextFactory.create(vCenterHost, userName, password); + context = VmwareContextFactory.create(vCenterHost, userName, password, _vCenterSessionTimeout); // Check if DC exists on vCenter try { diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java index 3ed5939aac55..10025b972057 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java @@ -55,7 +55,7 @@ void init() { s_clusterMgr = _clusterMgr; } - public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { assert (vCenterAddress != null); assert (vCenterUserName != null); assert (vCenterPassword != null); @@ -66,7 +66,7 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName StringUtils.getMaskedPasswordForDisplay(vCenterPassword)); VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.setVcenterSessionTimeout(s_vmwareMgr.getVcenterSessionTimeout()); + vimClient.setVcenterSessionTimeout(sessionTimeout); vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); VmwareContext context = new VmwareContext(vimClient, vCenterAddress); @@ -76,21 +76,21 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName()); context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId())); - context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName, sessionTimeout)); return context; } - public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { - VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName, sessionTimeout); if (context == null) { - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } else { // Validate current context and verify if vCenter session timeout value of the context matches the timeout value set by Admin - if (!context.validate() || (context.getVimClient().getVcenterSessionTimeout() != s_vmwareMgr.getVcenterSessionTimeout())) { + if (!context.validate()) { s_logger.info("Validation of the context failed, dispose and create a new one"); context.close(); - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index cfd565f0f94c..e1afda6d192a 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -328,7 +328,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa protected int _portsPerDvPortGroup; protected boolean _fullCloneFlag = false; protected boolean _instanceNameFlag = false; - + protected int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds protected boolean _recycleHungWorker = false; protected DiskControllerType _rootDiskController = DiskControllerType.ide; @@ -5571,6 +5571,10 @@ else if (value != null && value.equalsIgnoreCase("ide")) _instanceNameFlag = false; } + Integer sessionTimeout = (Integer)params.get("vmware.vcenter.session.timeout"); + if (sessionTimeout != null) + _vCenterSessionTimeout = sessionTimeout.intValue(); + value = (String)params.get("scripts.timeout"); int timeout = NumbersUtil.parseInt(value, 1440) * 1000; @@ -5625,9 +5629,16 @@ public VmwareHypervisorHost getHyperHost(VmwareContext context) { @Override public VmwareContext getServiceContext(Command cmd) { VmwareContext context = null; + int vCenterSessionTimeout; + if (cmd != null && cmd.getContextParam("vCenterSessionTimeout") != null) { + vCenterSessionTimeout = NumbersUtil.parseInt(cmd.getContextParam("vCenterSessionTimeout"), 1200000); + } else { + vCenterSessionTimeout = _vCenterSessionTimeout; + } + if(s_serviceContext.get() != null) { context = s_serviceContext.get(); - String poolKey = VmwareContextPool.composePoolKey(_vCenterAddress, _username); + String poolKey = VmwareContextPool.composePoolKey(_vCenterAddress, _username, vCenterSessionTimeout); // Before re-using the thread local context, ensure it corresponds to the right vCenter API session and that it is valid to make calls. if(context.getPoolKey().equals(poolKey)) { if (context.validate()) { @@ -5641,11 +5652,11 @@ public VmwareContext getServiceContext(Command cmd) { } } else { // Exisitng ThreadLocal context corresponds to a different vCenter API session. Why has it not been recycled? - s_logger.warn("ThreadLocal VMware context: " + poolKey + " doesn't correspond to the right vCenter. Expected VMware context: " + context.getPoolKey()); + s_logger.warn("ThreadLocal VMware context: " + poolKey + ". Expected VMware context: " + context.getPoolKey()); } } try { - context = VmwareContextFactory.getContext(_vCenterAddress, _username, _password); + context = VmwareContextFactory.getContext(_vCenterAddress, _username, _password, vCenterSessionTimeout); s_serviceContext.set(context); } catch (Exception e) { s_logger.error("Unable to connect to vSphere server: " + _vCenterAddress, e); diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java index 6e19ba67bf50..1fcd80235523 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java @@ -36,33 +36,33 @@ public static void initFactoryEnvironment() { s_pool = new VmwareContextPool(); } - public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { assert (vCenterAddress != null); assert (vCenterUserName != null); assert (vCenterPassword != null); String serviceUrl = "https://" + vCenterAddress + "/sdk/vimService"; VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.setVcenterSessionTimeout(s_vCenterSessionTimeout); + vimClient.setVcenterSessionTimeout(sessionTimeout); vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); VmwareContext context = new VmwareContext(vimClient, vCenterAddress); assert (context != null); - context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName, sessionTimeout)); return context; } - public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { - VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName, sessionTimeout); if (context == null) { - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } else { - // Validate current context and verify if vCenter session timeout value of the context matches the timeout value set by Admin - if (!context.validate() || (context.getVimClient().getVcenterSessionTimeout() != s_vCenterSessionTimeout)) { + // Validate current context + if (!context.validate()) { s_logger.info("Validation of the context faild. dispose and create a new one"); context.close(); - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index d0d5964bfa92..40099a51a8d6 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -46,10 +46,10 @@ import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostNetworkSummary; import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareContextPool; import com.cloud.serializer.GsonHelper; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; -import com.cloud.utils.StringUtils; public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageResourceHandler, VmwareHostService, VmwareStorageMount { private static final Logger s_logger = Logger.getLogger(VmwareSecondaryStorageResourceHandler.class); @@ -209,20 +209,31 @@ public VmwareContext getServiceContext(Command cmd) { _resource.ensureOutgoingRuleForAddress(vCenterAddress); VmwareContext context = currentContext.get(); - if (context != null && !context.validate()) { - invalidateServiceContext(context); - context = null; + if (context != null) { + String poolKey = VmwareContextPool.composePoolKey(vCenterAddress, username, vCenterSessionTimeout); + // Before re-using the thread local context, ensure it corresponds to the right vCenter API session with the expected timeout and that it is valid to make calls. + if(context.getPoolKey().equals(poolKey)) { + if (context.validate()) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("ThreadLocal context is still valid, just reuse"); + } + } else { + s_logger.info("Validation of the context failed, dispose and use a new one"); + invalidateServiceContext(context); + context = null; + } + } else { + s_logger.warn("ThreadLocal VMware context: " + poolKey + ". Expected VMware context: " + context.getPoolKey()); + context = null; + } } + if (context == null) { - s_logger.info("Open new VmwareContext. vCenter: " + vCenterAddress + ", user: " + username + ", password: " + StringUtils.getMaskedPasswordForDisplay(password)); - VmwareSecondaryStorageContextFactory.setVcenterSessionTimeout(vCenterSessionTimeout); - context = VmwareSecondaryStorageContextFactory.getContext(vCenterAddress, username, password); - } - if (context != null) { - context.registerStockObject("serviceconsole", cmd.getContextParam("serviceconsole")); - context.registerStockObject("manageportgroup", cmd.getContextParam("manageportgroup")); - context.registerStockObject("noderuninfo", cmd.getContextParam("noderuninfo")); + context = VmwareSecondaryStorageContextFactory.getContext(vCenterAddress, username, password, vCenterSessionTimeout); } + context.registerStockObject("serviceconsole", cmd.getContextParam("serviceconsole")); + context.registerStockObject("manageportgroup", cmd.getContextParam("manageportgroup")); + context.registerStockObject("noderuninfo", cmd.getContextParam("noderuninfo")); currentContext.set(context); return context; } catch (Exception e) { diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 551c61ea2e46..1e007b28a45f 100644 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -1210,6 +1210,7 @@ public enum Config { null), VmwareHungWorkerTimeout("Advanced", ManagementServer.class, Long.class, "vmware.hung.wokervm.timeout", "7200", "Worker VM timeout in seconds", null), VmwareVcenterSessionTimeout("Advanced", ManagementServer.class, Long.class, "vmware.vcenter.session.timeout", "1200", "VMware client timeout in seconds", null), + VmwareSnapshotBackupSessionTimeout("Advanced", ManagementServer.class, Long.class, "vmware.snapshot.backup.session.timeout", "1200", "VMware client timeout in seconds for snapshot backup", null), // Midonet MidoNetAPIServerAddress( diff --git a/setup/db/db/schema-4930to41000.sql b/setup/db/db/schema-4930to41000.sql index 59c452fba632..2ecade010b61 100644 --- a/setup/db/db/schema-4930to41000.sql +++ b/setup/db/db/schema-4930to41000.sql @@ -246,6 +246,9 @@ CREATE TABLE `cloud`.`guest_os_details` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; ALTER TABLE `user_ip_address` ADD COLUMN `rule_state` VARCHAR(32) COMMENT 'static rule state while removing'; + +INSERT IGNORE INTO `cloud`.`configuration` (category, instance, component, name, value, description, default_value) VALUES ('Advanced', 'DEFAULT', 'management-server', 'vmware.snapshot.backup.session.timeout', '1200', 'VMware client timeout in seconds for snapshot backup', '1200'); + CREATE TABLE `cloud`.`firewall_rules_dcidrs`( `id` BIGINT(20) unsigned NOT NULL AUTO_INCREMENT, `firewall_rule_id` BIGINT(20) unsigned NOT NULL, @@ -254,4 +257,4 @@ CREATE TABLE `cloud`.`firewall_rules_dcidrs`( UNIQUE KEY `unique_rule_dcidrs` (`firewall_rule_id`, `destination_cidr`), KEY `fk_firewall_dcidrs_firewall_rules` (`firewall_rule_id`), CONSTRAINT `fk_firewall_dcidrs_firewall_rules` FOREIGN KEY (`firewall_rule_id`) REFERENCES `firewall_rules` (`id`) ON DELETE CASCADE -)ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file +)ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java b/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java index b2e7636e0ae3..db6f7ca4b00e 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java @@ -56,8 +56,8 @@ public VmwareContextPool(int maxIdleQueueLength, Duration idleCheckInterval) { _timer.scheduleAtFixedRate(getTimerTask(), _idleCheckInterval.getMillis(), _idleCheckInterval.getMillis()); } - public VmwareContext getContext(final String vCenterAddress, final String vCenterUserName) { - final String poolKey = composePoolKey(vCenterAddress, vCenterUserName).intern(); + public VmwareContext getContext(final String vCenterAddress, final String vCenterUserName, int vCenterSessionTimeout) { + final String poolKey = composePoolKey(vCenterAddress, vCenterUserName, vCenterSessionTimeout).intern(); if (Strings.isNullOrEmpty(poolKey)) { return null; } @@ -158,9 +158,10 @@ private void doKeepAlive() { } } - public static String composePoolKey(final String vCenterAddress, final String vCenterUserName) { + public static String composePoolKey(final String vCenterAddress, final String vCenterUserName, int vCenterSessionTimeout) { assert (vCenterUserName != null); assert (vCenterAddress != null); - return vCenterUserName + "@" + vCenterAddress; + int sessionTimeoutInSeconds = vCenterSessionTimeout/1000; + return vCenterUserName + "@" + vCenterAddress + ";" + String.valueOf(sessionTimeoutInSeconds); } } diff --git a/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java b/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java index 3d0e73690172..bd3849914c68 100644 --- a/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java +++ b/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java @@ -37,7 +37,7 @@ public class TestVmwareContextFactory { s_pool = new VmwareContextPool(); } - public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { assert (vCenterAddress != null); assert (vCenterUserName != null); assert (vCenterPassword != null); @@ -49,20 +49,20 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName StringUtils.getMaskedPasswordForDisplay(vCenterPassword)); VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.setVcenterSessionTimeout(1200000); + vimClient.setVcenterSessionTimeout(sessionTimeout); vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); VmwareContext context = new VmwareContext(vimClient, vCenterAddress); return context; } - public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { - VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName, sessionTimeout); if (context == null) - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); if (context != null) { - context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName, sessionTimeout)); } return context; diff --git a/vmware-base/test/com/cloud/hypervisor/vmware/util/VmwareContextPoolTest.java b/vmware-base/test/com/cloud/hypervisor/vmware/util/VmwareContextPoolTest.java index 0c365db6fefe..48f3246a5c52 100644 --- a/vmware-base/test/com/cloud/hypervisor/vmware/util/VmwareContextPoolTest.java +++ b/vmware-base/test/com/cloud/hypervisor/vmware/util/VmwareContextPoolTest.java @@ -49,7 +49,7 @@ public void stop() { @Override public void run() { - final String poolKey = pool.composePoolKey(vmwareAddress, vmwareUsername); + final String poolKey = pool.composePoolKey(vmwareAddress, vmwareUsername, vCenterSessionTimeout); while (canRun) { pool.registerContext(createDummyContext(pool, poolKey)); counter++; @@ -61,6 +61,7 @@ public void run() { private VmwareContext vmwareContext; private String vmwareAddress = "address"; private String vmwareUsername = "username"; + private int vCenterSessionTimeout = 1200000; private int contextLength = 10; private Duration idleCheckInterval = Duration.millis(1000L); @@ -74,7 +75,7 @@ public VmwareContext createDummyContext(final VmwareContextPool pool, final Stri @Before public void setUp() throws Exception { - final String poolKey = vmwareContextPool.composePoolKey(vmwareAddress, vmwareUsername); + final String poolKey = vmwareContextPool.composePoolKey(vmwareAddress, vmwareUsername, vCenterSessionTimeout); vmwareContextPool = new VmwareContextPool(contextLength, idleCheckInterval); vmwareContext = createDummyContext(vmwareContextPool, poolKey); } @@ -82,18 +83,18 @@ public void setUp() throws Exception { @Test public void testRegisterContext() throws Exception { vmwareContextPool.registerContext(vmwareContext); - Assert.assertEquals(vmwareContextPool.getContext(vmwareAddress, vmwareUsername), vmwareContext); + Assert.assertEquals(vmwareContextPool.getContext(vmwareAddress, vmwareUsername, vCenterSessionTimeout), vmwareContext); } @Test public void testUnregisterContext() throws Exception { vmwareContextPool.unregisterContext(vmwareContext); - Assert.assertNull(vmwareContextPool.getContext(vmwareAddress, vmwareUsername)); + Assert.assertNull(vmwareContextPool.getContext(vmwareAddress, vmwareUsername, vCenterSessionTimeout)); } @Test public void testComposePoolKey() throws Exception { - Assert.assertEquals(vmwareContextPool.composePoolKey(vmwareAddress, vmwareUsername), vmwareUsername + "@" + vmwareAddress); + Assert.assertEquals(vmwareContextPool.composePoolKey(vmwareAddress, vmwareUsername, vCenterSessionTimeout), vmwareUsername + "@" + vmwareAddress + ";" + String.valueOf(vCenterSessionTimeout/1000)); } @Test