diff --git a/api/src/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/org/apache/cloudstack/api/response/TemplateResponse.java index 3e21043e3391..248b3f3c0cdb 100644 --- a/api/src/org/apache/cloudstack/api/response/TemplateResponse.java +++ b/api/src/org/apache/cloudstack/api/response/TemplateResponse.java @@ -179,6 +179,10 @@ public class TemplateResponse extends BaseResponse implements ControlledViewEnti @Param(description = "the list of resource tags associated with tempate", responseObject = ResourceTagResponse.class) private Set tags; + @SerializedName(ApiConstants.BITS) + @Param(description="the processor bit size", since = "4.10") + private int bits; + @SerializedName(ApiConstants.SSHKEY_ENABLED) @Param(description = "true if template is sshkey enabled, false otherwise") private Boolean sshKeyEnabled; @@ -361,4 +365,9 @@ public void setDynamicallyScalable(boolean isDynamicallyScalable) { public String getZoneId() { return zoneId; } + public void setBits(int bits) { + this.bits = bits; + } + + } diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VRScripts.java b/core/src/com/cloud/agent/resource/virtualnetwork/VRScripts.java index a2515056be1d..838f0877918f 100644 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VRScripts.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VRScripts.java @@ -19,6 +19,8 @@ package com.cloud.agent.resource.virtualnetwork; +import org.joda.time.Duration; + public class VRScripts { public final static String CONFIG_PERSIST_LOCATION = "/var/cache/cloud/"; public final static String IP_ASSOCIATION_CONFIG = "ip_associations.json"; @@ -40,7 +42,8 @@ public class VRScripts { public static final String LOAD_BALANCER_CONFIG = "load_balancer.json"; public final static String CONFIG_CACHE_LOCATION = "/var/cache/cloud/"; - public final static int DEFAULT_EXECUTEINVR_TIMEOUT = 120; //Seconds + public final static Duration VR_SCRIPT_EXEC_TIMEOUT = Duration.standardMinutes(10); + public final static Duration CONNECTION_TIMEOUT = Duration.standardMinutes(1); // New scripts for use with chef public static final String UPDATE_CONFIG = "update_config.py"; diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRouterDeployer.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRouterDeployer.java index f6a1694c558a..6501292b82b9 100644 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRouterDeployer.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRouterDeployer.java @@ -19,13 +19,14 @@ package com.cloud.agent.resource.virtualnetwork; +import org.joda.time.Duration; + import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.utils.ExecutionResult; public interface VirtualRouterDeployer { ExecutionResult executeInVR(String routerIp, String script, String args); - /* timeout in seconds */ - ExecutionResult executeInVR(String routerIp, String script, String args, int timeout); + ExecutionResult executeInVR(String routerIp, String script, String args, Duration timeout); ExecutionResult createFileInVR(String routerIp, String path, String filename, String content); ExecutionResult prepareCommand(NetworkElementCommand cmd); ExecutionResult cleanupCommand(NetworkElementCommand cmd); diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index f3edc6967594..87a38d3e39fb 100644 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.SocketChannel; +import org.joda.time.Duration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -73,7 +74,7 @@ public class VirtualRoutingResource { private int _sleep; private int _retry; private int _port; - private int _eachTimeout; + private Duration _eachTimeout; private String _cfgVersion = "1.0"; @@ -153,10 +154,10 @@ private Answer executeQueryCommand(NetworkElementCommand cmd) { } private ExecutionResult applyConfigToVR(String routerAccessIp, ConfigItem c) { - return applyConfigToVR(routerAccessIp, c, VRScripts.DEFAULT_EXECUTEINVR_TIMEOUT); + return applyConfigToVR(routerAccessIp, c, VRScripts.VR_SCRIPT_EXEC_TIMEOUT); } - private ExecutionResult applyConfigToVR(String routerAccessIp, ConfigItem c, int timeout) { + private ExecutionResult applyConfigToVR(String routerAccessIp, ConfigItem c, Duration timeout) { if (c instanceof FileConfigItem) { FileConfigItem configItem = (FileConfigItem)c; return _vrDeployer.createFileInVR(routerAccessIp, configItem.getFilePath(), configItem.getFileName(), configItem.getFileContents()); @@ -271,7 +272,7 @@ public boolean configure(final String name, final Map params) th _port = NumbersUtil.parseInt(value, 3922); value = (String)params.get("router.aggregation.command.each.timeout"); - _eachTimeout = NumbersUtil.parseInt(value, 3); + _eachTimeout = Duration.standardSeconds(NumbersUtil.parseInt(value, 10)); if (_vrDeployer == null) { throw new ConfigurationException("Unable to find the resource for VirtualRouterDeployer!"); @@ -374,9 +375,9 @@ private Answer execute(AggregationControlCommand cmd) { FileConfigItem fileConfigItem = new FileConfigItem(VRScripts.CONFIG_CACHE_LOCATION, cfgFileName, sb.toString()); ScriptConfigItem scriptConfigItem = new ScriptConfigItem(VRScripts.VR_CFG, "-c " + VRScripts.CONFIG_CACHE_LOCATION + cfgFileName); // 120s is the minimal timeout - int timeout = answerCounts * _eachTimeout; - if (timeout < 120) { - timeout = 120; + Duration timeout = _eachTimeout.withDurationAdded(_eachTimeout.getStandardSeconds(), answerCounts); + if (timeout.isShorterThan(VRScripts.VR_SCRIPT_EXEC_TIMEOUT)) { + timeout = VRScripts.VR_SCRIPT_EXEC_TIMEOUT; } ExecutionResult result = applyConfigToVR(cmd.getRouterAccessIp(), fileConfigItem); diff --git a/core/test/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java b/core/test/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java index 6b5f1d1baf5f..6405037dd0fd 100644 --- a/core/test/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java +++ b/core/test/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java @@ -31,6 +31,7 @@ import javax.naming.ConfigurationException; +import org.joda.time.Duration; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -97,11 +98,11 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { @Override public ExecutionResult executeInVR(final String routerIp, final String script, final String args) { - return executeInVR(routerIp, script, args, 60); + return executeInVR(routerIp, script, args, Duration.standardSeconds(60L)); } @Override - public ExecutionResult executeInVR(final String routerIp, final String script, final String args, final int timeout) { + public ExecutionResult executeInVR(final String routerIp, final String script, final String args, final Duration timeout) { assertEquals(routerIp, ROUTERIP); verifyCommand(_currentCmd, script, args); return new ExecutionResult(true, null); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index e541cb71fe82..77f229c5204a 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -267,7 +267,9 @@ public boolean deleteSnapshot(Long snapshotId) { SnapshotDataStoreVO snapshotOnPrimary = snapshotStoreDao.findBySnapshot(snapshotId, DataStoreRole.Primary); if (snapshotOnPrimary != null) { SnapshotInfo snapshotOnPrimaryInfo = snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Primary); - if (((PrimaryDataStoreImpl)snapshotOnPrimaryInfo.getDataStore()).getPoolType() == StoragePoolType.RBD) { + long volumeId = snapshotOnPrimary.getVolumeId(); + VolumeVO volumeVO = volumeDao.findById(volumeId); + if (((PrimaryDataStoreImpl)snapshotOnPrimaryInfo.getDataStore()).getPoolType() == StoragePoolType.RBD && volumeVO != null) { snapshotSvr.deleteSnapshot(snapshotOnPrimaryInfo); } snapshotOnPrimary.setState(State.Destroyed); diff --git a/packaging/systemd/cloudstack-usage.service b/packaging/systemd/cloudstack-usage.service index 9a1827da6132..d80aba6f6fee 100644 --- a/packaging/systemd/cloudstack-usage.service +++ b/packaging/systemd/cloudstack-usage.service @@ -27,10 +27,11 @@ Environment=JAVA_HOME=/usr/lib/jvm/jre Environment=JAVA_HEAP_INITIAL=256m Environment=JAVA_HEAP_MAX=2048m Environment=JAVA_CLASS=com.cloud.usage.UsageServer +Environment=JAVA_PID=$$ ExecStart=/bin/sh -ec '\ export UCP=`ls /usr/share/cloudstack-usage/cloud-usage-*.jar /usr/share/cloudstack-usage/lib/*.jar | tr "\\n" ":"`; \ export CLASSPATH="$UCP:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar"; \ - ${JAVA_HOME}/bin/java -Xms${JAVA_HEAP_INITIAL} -Xmx${JAVA_HEAP_MAX} -cp "$CLASSPATH" $JAVA_CLASS' + ${JAVA_HOME}/bin/java -Dpid=${JAVA_PID} -Xms${JAVA_HEAP_INITIAL} -Xmx${JAVA_HEAP_MAX} -cp "$CLASSPATH" $JAVA_CLASS' Restart=always RestartSec=10s diff --git a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java index 5b5ec57a8897..862cc30a2e5b 100644 --- a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java +++ b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/resource/HypervDirectConnectResource.java @@ -32,6 +32,7 @@ import java.security.UnrecoverableKeyException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import org.joda.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -622,11 +623,11 @@ private UnPlugNicAnswer execute(final UnPlugNicCommand cmd) { @Override public ExecutionResult executeInVR(final String routerIP, final String script, final String args) { - return executeInVR(routerIP, script, args, 120); + return executeInVR(routerIP, script, args, Duration.standardSeconds(120L)); } @Override - public ExecutionResult executeInVR(final String routerIP, final String script, final String args, final int timeout) { + public ExecutionResult executeInVR(final String routerIP, final String script, final String args, final Duration timeout) { Pair result; //TODO: Password should be masked, cannot output to log directly @@ -635,8 +636,8 @@ public ExecutionResult executeInVR(final String routerIP, final String script, f } try { - result = SshHelper.sshExecute(routerIP, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/opt/cloud/bin/" + script + " " + args, - 60000, 60000, timeout * 1000); + result = SshHelper.sshExecute(routerIP, DEFAULT_DOMR_SSHPORT, "root", getSystemVMKeyFile(), null, "/opt/cloud/bin/" + script + " " + args, VRScripts.CONNECTION_TIMEOUT, + VRScripts.CONNECTION_TIMEOUT, timeout); } catch (final Exception e) { final String msg = "Command failed due to " + e ; s_logger.error(msg); diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 883c8c99429d..2b1cf6881be3 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; +import org.joda.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -189,7 +190,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private long _hvVersion; private long _kernelVersion; - private int _timeout; + private Duration _timeout; + private static final int NUMMEMSTATS =2; private KVMHAMonitor _monitor; public static final String SSHKEYSPATH = "/root/.ssh"; @@ -276,12 +278,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv @Override public ExecutionResult executeInVR(final String routerIp, final String script, final String args) { - return executeInVR(routerIp, script, args, _timeout / 1000); + return executeInVR(routerIp, script, args, _timeout); } @Override - public ExecutionResult executeInVR(final String routerIp, final String script, final String args, final int timeout) { - final Script command = new Script(_routerProxyPath, timeout * 1000, s_logger); + public ExecutionResult executeInVR(final String routerIp, final String script, final String args, final Duration timeout) { + final Script command = new Script(_routerProxyPath, timeout, s_logger); final AllLinesParser parser = new AllLinesParser(); command.add(script); command.add(routerIp); @@ -383,7 +385,7 @@ public String getUpdateHostPasswdPath() { return _updateHostPasswdPath; } - public int getTimeout() { + public Duration getTimeout() { return _timeout; } @@ -774,7 +776,7 @@ public boolean configure(final String name, final Map params) th } value = (String)params.get("scripts.timeout"); - _timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000; + _timeout = Duration.standardSeconds(NumbersUtil.parseInt(value, 30 * 60)); value = (String)params.get("stop.script.timeout"); _stopTimeout = NumbersUtil.parseInt(value, 120) * 1000; diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsCreateTunnelCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsCreateTunnelCommandWrapper.java index b840e8b472c3..03d9cf94561f 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsCreateTunnelCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsCreateTunnelCommandWrapper.java @@ -29,7 +29,7 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.utils.script.Script; -@ResourceWrapper(handles = OvsCreateTunnelCommand.class) +@ResourceWrapper(handles = OvsCreateTunnelCommand.class) public final class LibvirtOvsCreateTunnelCommandWrapper extends CommandWrapper { private static final Logger s_logger = Logger.getLogger(LibvirtOvsCreateTunnelCommandWrapper.class); @@ -40,13 +40,10 @@ public Answer execute(final OvsCreateTunnelCommand command, final LibvirtComputi try { if (!libvirtComputingResource.findOrCreateTunnelNetwork(bridge)) { s_logger.debug("Error during bridge setup"); - return new OvsCreateTunnelAnswer(command, false, - "Cannot create network", bridge); + return new OvsCreateTunnelAnswer(command, false, "Cannot create network", bridge); } - libvirtComputingResource.configureTunnelNetwork(command.getNetworkId(), command.getFrom(), - command.getNetworkName()); - + libvirtComputingResource.configureTunnelNetwork(command.getNetworkId(), command.getFrom(), command.getNetworkName()); final Script scriptCommand = new Script(libvirtComputingResource.getOvsTunnelPath(), libvirtComputingResource.getTimeout(), s_logger); scriptCommand.add("create_tunnel"); scriptCommand.add("--bridge", bridge); @@ -57,8 +54,7 @@ public Answer execute(final OvsCreateTunnelCommand command, final LibvirtComputi final String result = scriptCommand.execute(); if (result != null) { - return new OvsCreateTunnelAnswer(command, true, result, null, - bridge); + return new OvsCreateTunnelAnswer(command, true, result, null, bridge); } else { return new OvsCreateTunnelAnswer(command, false, result, bridge); } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPvlanSetupCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPvlanSetupCommandWrapper.java index 435b98a18ab0..3e01dc425993 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPvlanSetupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPvlanSetupCommandWrapper.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.log4j.Logger; +import org.joda.time.Duration; import org.libvirt.Connect; import org.libvirt.LibvirtException; @@ -45,8 +46,8 @@ public Answer execute(final PvlanSetupCommand command, final LibvirtComputingRes final String op = command.getOp(); final String dhcpName = command.getDhcpName(); final String dhcpMac = command.getDhcpMac(); - final String dhcpIp = command.getDhcpIp(); final String vmMac = command.getVmMac(); + final String dhcpIp = command.getDhcpIp(); boolean add = true; String opr = "-A"; @@ -58,7 +59,7 @@ public Answer execute(final PvlanSetupCommand command, final LibvirtComputingRes String result = null; try { final String guestBridgeName = libvirtComputingResource.getGuestBridgeName(); - final int timeout = libvirtComputingResource.getTimeout(); + final Duration timeout = libvirtComputingResource.getTimeout(); if (command.getType() == PvlanSetupCommand.Type.DHCP) { final String ovsPvlanDhcpHostPath = libvirtComputingResource.getOvsPvlanDhcpHostPath(); diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index a10b477e4113..3acea5ea2d67 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -53,6 +53,7 @@ import org.apache.cloudstack.utils.linux.MemStat; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.commons.lang.SystemUtils; +import org.joda.time.Duration; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -1623,7 +1624,7 @@ public void testModifySshKeysCommand() { when(libvirtUtilitiesHelper.retrieveSshPubKeyPath()).thenReturn("/path/pub/keys"); when(libvirtUtilitiesHelper.retrieveSshPrvKeyPath()).thenReturn("/path/pvt/keys"); - when(libvirtComputingResource.getTimeout()).thenReturn(0); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -2179,7 +2180,7 @@ public void testOvsVpcPhysicalTopologyConfigCommand() { final OvsVpcPhysicalTopologyConfigCommand command = new OvsVpcPhysicalTopologyConfigCommand(hosts, tiers, vms, cidr); when(libvirtComputingResource.getOvsTunnelPath()).thenReturn("/path"); - when(libvirtComputingResource.getTimeout()).thenReturn(0); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); @@ -2223,7 +2224,7 @@ public void testOvsVpcRoutingPolicyConfigCommand() { final OvsVpcRoutingPolicyConfigCommand command = new OvsVpcRoutingPolicyConfigCommand(id, cidr, acls, tiers); when(libvirtComputingResource.getOvsTunnelPath()).thenReturn("/path"); - when(libvirtComputingResource.getTimeout()).thenReturn(0); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); @@ -2699,6 +2700,7 @@ public void testOvsCreateTunnelCommand() { when(libvirtComputingResource.findOrCreateTunnelNetwork(bridge)).thenReturn(true); when(libvirtComputingResource.configureTunnelNetwork(command.getNetworkId(), command.getFrom(), command.getNetworkName())).thenReturn(true); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance(); assertNotNull(wrapper); @@ -4273,8 +4275,7 @@ public void testPvlanSetupCommandDhcpAdd() { final String guestBridgeName = "br0"; when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName); - final int timeout = 0; - when(libvirtComputingResource.getTimeout()).thenReturn(timeout); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final String ovsPvlanDhcpHostPath = "/pvlan"; when(libvirtComputingResource.getOvsPvlanDhcpHostPath()).thenReturn(ovsPvlanDhcpHostPath); when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); @@ -4315,8 +4316,7 @@ public void testPvlanSetupCommandVm() { final String guestBridgeName = "br0"; when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName); - final int timeout = 0; - when(libvirtComputingResource.getTimeout()).thenReturn(timeout); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final String ovsPvlanVmPath = "/pvlan"; when(libvirtComputingResource.getOvsPvlanVmPath()).thenReturn(ovsPvlanVmPath); @@ -4345,8 +4345,7 @@ public void testPvlanSetupCommandDhcpException() { final String guestBridgeName = "br0"; when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName); - final int timeout = 0; - when(libvirtComputingResource.getTimeout()).thenReturn(timeout); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final String ovsPvlanDhcpHostPath = "/pvlan"; when(libvirtComputingResource.getOvsPvlanDhcpHostPath()).thenReturn(ovsPvlanDhcpHostPath); when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); @@ -4387,8 +4386,7 @@ public void testPvlanSetupCommandDhcpDelete() { final String guestBridgeName = "br0"; when(libvirtComputingResource.getGuestBridgeName()).thenReturn(guestBridgeName); - final int timeout = 0; - when(libvirtComputingResource.getTimeout()).thenReturn(timeout); + when(libvirtComputingResource.getTimeout()).thenReturn(Duration.ZERO); final String ovsPvlanDhcpHostPath = "/pvlan"; when(libvirtComputingResource.getOvsPvlanDhcpHostPath()).thenReturn(ovsPvlanDhcpHostPath); when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3VirtualRoutingResource.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3VirtualRoutingResource.java index 46c60763d274..f99169d85653 100644 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3VirtualRoutingResource.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3VirtualRoutingResource.java @@ -17,6 +17,8 @@ package com.cloud.hypervisor.ovm3.resources; +import org.joda.time.Duration; + import javax.ejb.Local; import org.apache.log4j.Logger; @@ -27,6 +29,7 @@ import com.cloud.agent.api.routing.NetworkElementCommand; import com.cloud.agent.api.routing.SetSourceNatCommand; import com.cloud.agent.api.to.IpAddressTO; +import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer; import com.cloud.hypervisor.ovm3.objects.CloudstackPlugin; import com.cloud.hypervisor.ovm3.objects.Connection; @@ -38,7 +41,6 @@ public class Ovm3VirtualRoutingResource implements VirtualRouterDeployer { private final Logger logger = Logger .getLogger(Ovm3VirtualRoutingResource.class); private String domRCloudPath = "/opt/cloud/bin/"; - private int vrTimeout = 600; private Connection c; private String agentName; public Ovm3VirtualRoutingResource() { @@ -53,12 +55,12 @@ public void setConnection(Connection conn) { @Override public ExecutionResult executeInVR(String routerIp, String script, String args) { - return executeInVR(routerIp, script, args, vrTimeout); + return executeInVR(routerIp, script, args, VRScripts.VR_SCRIPT_EXEC_TIMEOUT); } @Override public ExecutionResult executeInVR(String routerIp, String script, - String args, int timeout) { + String args, Duration timeout) { if (!script.contains(domRCloudPath)) { script = domRCloudPath + "/" + script; } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index dd419f2574ff..8979e5245144 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -24,6 +24,7 @@ import java.net.URI; import java.nio.channels.SocketChannel; import java.rmi.RemoteException; +import org.joda.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -210,6 +211,7 @@ import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.api.to.VolumeTO; +import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer; import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource; import com.cloud.dc.DataCenter.NetworkType; @@ -1197,11 +1199,11 @@ private ExecutionResult prepareNetworkElementCommand(IpAssocCommand cmd) { @Override public ExecutionResult executeInVR(String routerIP, String script, String args) { - return executeInVR(routerIP, script, args, 120); + return executeInVR(routerIP, script, args, VRScripts.VR_SCRIPT_EXEC_TIMEOUT); } @Override - public ExecutionResult executeInVR(String routerIP, String script, String args, int timeout) { + public ExecutionResult executeInVR(String routerIP, String script, String args, Duration timeout) { Pair result; //TODO: Password should be masked, cannot output to log directly @@ -1212,7 +1214,7 @@ public ExecutionResult executeInVR(String routerIP, String script, String args, try { VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME); result = SshHelper.sshExecute(routerIP, DefaultDomRSshPort, "root", mgr.getSystemVMKeyFile(), null, "/opt/cloud/bin/" + script + " " + args, - 60000, 60000, timeout * 1000); + VRScripts.CONNECTION_TIMEOUT, VRScripts.CONNECTION_TIMEOUT, timeout); } catch (Exception e) { String msg = "Command failed due to " + VmwareHelper.getExceptionMessage(e); s_logger.error(msg); diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 7c8bca7f1bdb..ed86f75e496e 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -29,6 +29,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.charset.Charset; +import org.joda.time.Duration; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -97,6 +98,7 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer; import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource; import com.cloud.exception.InternalErrorException; @@ -1652,18 +1654,18 @@ public StopAnswer execute(final StopCommand cmd) { @Override public ExecutionResult executeInVR(final String routerIP, final String script, final String args) { // Timeout is 120 seconds by default - return executeInVR(routerIP, script, args, 120); + return executeInVR(routerIP, script, args, VRScripts.VR_SCRIPT_EXEC_TIMEOUT); } @Override - public ExecutionResult executeInVR(final String routerIP, final String script, final String args, final int timeout) { + public ExecutionResult executeInVR(final String routerIP, final String script, final String args, final Duration timeout) { Pair result; String cmdline = "/opt/cloud/bin/router_proxy.sh " + script + " " + routerIP + " " + args; // semicolon need to be escape for bash cmdline = cmdline.replaceAll(";", "\\\\;"); try { s_logger.debug("Executing command in VR: " + cmdline); - result = SshHelper.sshExecute(_host.getIp(), 22, _username, null, _password.peek(), cmdline, 60000, 60000, timeout * 1000); + result = SshHelper.sshExecute(_host.getIp(), 22, _username, null, _password.peek(), cmdline, VRScripts.CONNECTION_TIMEOUT, VRScripts.CONNECTION_TIMEOUT, timeout); } catch (final Exception e) { return new ExecutionResult(false, e.getMessage()); } diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index 689ae9c67520..5c2c7ba5b1e7 100644 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -147,6 +147,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InterruptedIOException; +import java.lang.reflect.Type; import java.net.InetAddress; import java.net.ServerSocket; import java.net.Socket; @@ -174,6 +175,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.gson.reflect.TypeToken; @Component public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiServerService { @@ -264,10 +266,11 @@ private void handleAsyncJobPublishEvent(String subject, String senderAddress, Ob String info = job.getCmdInfo(); String cmdEventType = "unknown"; if (info != null) { - String marker = "\"cmdEventType\""; - int begin = info.indexOf(marker); - if (begin >= 0) { - cmdEventType = info.substring(begin + marker.length() + 2, info.indexOf(",", begin) - 1); + Type type = new TypeToken>(){}.getType(); + Map cmdInfo = ApiGsonHelper.getBuilder().create().fromJson(info, type); + String eventTypeObj = cmdInfo.get("cmdEventType"); + if (eventTypeObj != null) { + cmdEventType = eventTypeObj; if (s_logger.isDebugEnabled()) s_logger.debug("Retrieved cmdEventType from job info: " + cmdEventType); diff --git a/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index 5444eca6ba56..dc6419e58b0f 100644 --- a/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -291,6 +291,7 @@ public TemplateResponse newIsoResponse(TemplateJoinVO iso) { isoResponse.setOsTypeId(iso.getGuestOSUuid()); isoResponse.setOsTypeName(iso.getGuestOSName()); + isoResponse.setBits(iso.getBits()); // populate owner. ApiResponseHelper.populateOwner(isoResponse, iso); diff --git a/server/src/com/cloud/network/IpAddressManagerImpl.java b/server/src/com/cloud/network/IpAddressManagerImpl.java index d4da5fae24c2..bd59c66df3ab 100644 --- a/server/src/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/com/cloud/network/IpAddressManagerImpl.java @@ -1698,6 +1698,22 @@ public String acquireGuestIpAddress(Network network, String requestedIp) { Random _rand = new Random(System.currentTimeMillis()); + /** + * Get the list of public IPs that need to be applied for a static NAT enable/disable operation. + * Manipulating only these ips prevents concurrency issues when disabling static nat at the same time. + * @param staticNats + * @return The list of IPs that need to be applied for the static NAT to work. + */ + public List getStaticNatSourceIps(List staticNats) { + List userIps = new ArrayList<>(); + + for (StaticNat snat : staticNats) { + userIps.add(_ipAddressDao.findById(snat.getSourceIpAddressId())); + } + + return userIps; + } + @Override public boolean applyStaticNats(List staticNats, boolean continueOnError, boolean forRevoke) throws ResourceUnavailableException { if (staticNats == null || staticNats.size() == 0) { @@ -1714,8 +1730,8 @@ public boolean applyStaticNats(List staticNats, boolean con return true; } - // get the list of public ip's owned by the network - List userIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), null); + List userIps = getStaticNatSourceIps(staticNats); + List publicIps = new ArrayList(); if (userIps != null && !userIps.isEmpty()) { for (IPAddressVO userIp : userIps) { diff --git a/server/src/com/cloud/network/router/CommandSetupHelper.java b/server/src/com/cloud/network/router/CommandSetupHelper.java index 7208b2568139..500fc49a0c0d 100644 --- a/server/src/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/com/cloud/network/router/CommandSetupHelper.java @@ -780,10 +780,11 @@ public int compare(final PublicIpAddress o1, final PublicIpAddress o2) { final boolean add = ipAddr.getState() == IpAddress.State.Releasing ? false : true; boolean sourceNat = ipAddr.isSourceNat(); - /* enable sourceNAT for the first ip of the public interface */ - if (firstIP) { - sourceNat = true; + /* enable sourceNAT for the first ip of the public interface as long as it's source nat. */ + if (firstIP && !sourceNat) { + firstIP = false; } + final String vlanId = ipAddr.getVlanTag(); final String vlanGateway = ipAddr.getGateway(); final String vlanNetmask = ipAddr.getNetmask(); diff --git a/server/test/com/cloud/network/IpAddressManagerTest.java b/server/test/com/cloud/network/IpAddressManagerTest.java new file mode 100644 index 000000000000..0bf92ee2f692 --- /dev/null +++ b/server/test/com/cloud/network/IpAddressManagerTest.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.network; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.rules.StaticNat; +import com.cloud.network.rules.StaticNatImpl; +import com.cloud.utils.net.Ip; + +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; + +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.mock; + +public class IpAddressManagerTest { + + @Mock + IPAddressDao _ipAddrDao; + + @InjectMocks + IpAddressManagerImpl _ipManager; + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testGetStaticNatSourceIps() { + String publicIpAddress = "192.168.1.3"; + IPAddressVO vo = mock(IPAddressVO.class); + when(vo.getAddress()).thenReturn(new Ip(publicIpAddress)); + when(vo.getId()).thenReturn(1l); + + when(_ipAddrDao.findById(anyLong())).thenReturn(vo); + StaticNat snat = new StaticNatImpl(1, 1, 1, 1, publicIpAddress, false); + + List ips = _ipManager.getStaticNatSourceIps(Collections.singletonList(snat)); + Assert.assertNotNull(ips); + Assert.assertEquals(1, ips.size()); + + IPAddressVO returnedVO = ips.get(0); + Assert.assertEquals(vo, returnedVO); + } +} diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py index 3f102e6e28d0..f4cff6fb1d24 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDhcp.py @@ -103,12 +103,10 @@ def delete_leases(self): return def preseed(self): - self.add_host("127.0.0.1", "localhost") + self.add_host("127.0.0.1", "localhost %s" % CsHelper.get_hostname()) self.add_host("::1", "localhost ip6-localhost ip6-loopback") self.add_host("ff02::1", "ip6-allnodes") self.add_host("ff02::2", "ip6-allrouters") - if self.config.is_vpc(): - self.add_host("127.0.0.1", CsHelper.get_hostname()) if self.config.is_router(): self.add_host(self.config.address().get_guest_ip(), "%s data-server" % CsHelper.get_hostname()) diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py index e30c012f10df..a5cb3e565319 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_forwardingrules.py @@ -64,6 +64,10 @@ def merge(dbag, rules): print "removing index %s" % str(index) if not index == -1: del dbag[source_ip][index] + # If all forwarding rules have been deleted + # remove IP from databag + if dbag[source_ip] == []: + del dbag[source_ip] return dbag diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py old mode 100755 new mode 100644 index efcf311296fa..050a8dc71e68 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs_ip.py @@ -15,18 +15,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pprint import pprint -from netaddr import * +from netaddr import * def merge(dbag, ip): - added = False + index = -1 # a non-valid array index for dev in dbag: if dev == "id": continue - for address in dbag[dev]: + for i, address in enumerate(dbag[dev]): if address['public_ip'] == ip['public_ip']: - dbag[dev].remove(address) + index = i ipo = IPNetwork(ip['public_ip'] + '/' + ip['netmask']) ip['device'] = 'eth' + str(ip['nic_dev_id']) @@ -37,8 +36,11 @@ def merge(dbag, ip): if 'nw_type' not in ip.keys(): ip['nw_type'] = 'public' if ip['nw_type'] == 'control': - dbag['eth' + str(ip['nic_dev_id'])] = [ip] + dbag[ip['device']] = [ip] else: - dbag.setdefault('eth' + str(ip['nic_dev_id']), []).append(ip) + if index != -1: + dbag[ip['device']][index] = ip + else: + dbag.setdefault(ip['device'], []).append(ip) return dbag diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index b8cb4fe8ce46..760847514175 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -39,6 +39,7 @@ import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; +import org.joda.time.Duration; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -64,10 +65,15 @@ public class Script implements Callable { Process _process; Thread _thread; - public int getExitValue() { + public int getExitValue() { return _process.exitValue(); } + public Script(String command, Duration timeout, Logger logger) { + this(command, timeout.getMillis(), logger); + } + + @Deprecated public Script(String command, long timeout, Logger logger) { _command = new ArrayList(); _command.add(command); @@ -80,6 +86,11 @@ public Script(String command, long timeout, Logger logger) { _logger = logger != null ? logger : s_logger; } + public Script(boolean runWithSudo, String command, Duration timeout, Logger logger) { + this(runWithSudo, command, timeout.getMillis(), logger); + } + + @Deprecated public Script(boolean runWithSudo, String command, long timeout, Logger logger) { this(command, timeout, logger); if (runWithSudo) { @@ -95,6 +106,11 @@ public Script(String command) { this(command, 0, s_logger); } + public Script(String command, Duration timeout) { + this(command, timeout.getMillis(), s_logger); + } + + @Deprecated public Script(String command, long timeout) { this(command, timeout, s_logger); } diff --git a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java index d3c88c8367ad..8889016d0559 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java @@ -23,6 +23,7 @@ import java.io.InputStream; import org.apache.log4j.Logger; +import org.joda.time.Duration; import com.trilead.ssh2.ChannelCondition; @@ -122,8 +123,13 @@ public static void scpTo(String host, int port, String user, File pemKeyFile, St } } - public static Pair sshExecute(String host, int port, String user, File pemKeyFile, String password, String command, int connectTimeoutInMs, - int kexTimeoutInMs, int waitResultTimeoutInMs) throws Exception { + public static Pair sshExecute(String host, int port, String user, File pemKeyFile, String password, String command, Duration connectTimeout, + Duration kexTimeout, Duration waitTime) throws Exception { + return sshExecute(host, port, user, pemKeyFile, password, command, (int)connectTimeout.getMillis(), (int)kexTimeout.getMillis(), (int)waitTime.getMillis()); + } + + public static Pair sshExecute(String host, int port, String user, File pemKeyFile, String password, String command, int connectTimeoutInMs, int kexTimeoutInMs, + int waitResultTimeoutInMs) throws Exception { com.trilead.ssh2.Connection conn = null; com.trilead.ssh2.Session sess = null;