From 86093ec8f8d5275903314077513468753a73fa04 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 13 May 2019 07:57:46 +0000 Subject: [PATCH 1/5] KVM: Propagating changes on host parameters to the agents --- .../cloud/agent/manager/AgentManagerImpl.java | 60 +++++++++++++++++++ .../resource/LibvirtComputingResource.java | 19 ++++++ .../LibvirtSetHostParamsCommandWrapper.java | 1 + 3 files changed, 80 insertions(+) diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 68e27211603b..dc9f93006f0e 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -45,6 +45,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; @@ -83,6 +85,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; +import com.cloud.event.EventTypes; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConnectionException; import com.cloud.exception.OperationTimedoutException; @@ -187,6 +190,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl ResourceManager _resourceMgr; @Inject ManagementServiceConfiguration mgmtServiceConf; + @Inject + MessageBus messageBus; protected final ConfigKey Workers = new ConfigKey("Advanced", Integer.class, "workers", "5", "Number of worker threads handling remote agent connections.", false); @@ -237,6 +242,8 @@ public boolean configure(final String name, final Map params) th _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor")); + initMessageBusListener(); + return true; } @@ -1788,4 +1795,57 @@ public int getTimeout() { } + protected Map> getHostsPerZone() { + List allHosts = _resourceMgr.listAllHostsInAllZonesByType(Host.Type.Routing); + if (allHosts == null) { + return null; + } + Map> hostsByZone = new HashMap>(); + for (HostVO host : allHosts) { + if (host.getHypervisorType() == HypervisorType.KVM || host.getHypervisorType() == HypervisorType.LXC) { + Long zoneId = host.getDataCenterId(); + List hostIds = hostsByZone.get(zoneId); + if (hostIds == null) { + hostIds = new ArrayList(); + } + hostIds.add(host.getId()); + hostsByZone.put(zoneId, hostIds); + } + } + return hostsByZone; + } + + private void sendCommandToAgents(Map> hostsPerZone, Map params) { + SetHostParamsCommand cmds = new SetHostParamsCommand(params); + for (Long zoneId : hostsPerZone.keySet()) { + List hostIds = hostsPerZone.get(zoneId); + for (Long hostId : hostIds) { + Answer answer = easySend(hostId, cmds); + if (answer == null || !answer.getResult()) { + s_logger.error("Error sending parameters to agent " + hostId); + } + } + } + } + + private void propagateChangeToAgents() { + s_logger.debug("Propagating changes on host parameters to the agents"); + Map> hostsPerZone = getHostsPerZone(); + Map params = new HashMap(); + params.put("router.aggregation.command.each.timeout", _configDao.getValue("router.aggregation.command.each.timeout")); + sendCommandToAgents(hostsPerZone, params); + } + + private void initMessageBusListener() { + messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() { + @Override + public void onPublishMessage(String serderAddress, String subject, Object args) { + String globalSettingUpdated = (String)args; + if (globalSettingUpdated.equals("router.aggregation.command.each.timeout")) { + propagateChangeToAgents(); + } + } + }); + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 90eafadc1cd6..9f56e1c0be86 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -106,6 +106,7 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.dao.impl.PropertiesStorage; import com.cloud.agent.resource.virtualnetwork.VRScripts; import com.cloud.agent.resource.virtualnetwork.VirtualRouterDeployer; import com.cloud.agent.resource.virtualnetwork.VirtualRoutingResource; @@ -1094,6 +1095,24 @@ public boolean configure(final String name, final Map params) th return true; } + public boolean configureHostParams(final Map params) { + final File file = PropertiesUtil.findConfigFile("agent.properties"); + if (file == null) { + s_logger.error("Unable to find the file agent.properties"); + return false; + } + // Save configurations in agent.properties + PropertiesStorage storage = new PropertiesStorage(); + storage.configure("Storage", new HashMap()); + if (params.get("router.aggregation.command.each.timeout") == null) { + String value = (String)params.get("router.aggregation.command.each.timeout"); + Integer intValue = NumbersUtil.parseInt(value, 10); + storage.persist("router.aggregation.command.each.timeout", String.valueOf(intValue)); + } + + return true; + } + protected void configureDiskActivityChecks(final Map params) { _diskActivityCheckEnabled = Boolean.parseBoolean((String)params.get("vm.diskactivity.checkenabled")); if (_diskActivityCheckEnabled) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetHostParamsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetHostParamsCommandWrapper.java index 52dd0e9e4d4c..e07f0a6b01bf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetHostParamsCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetHostParamsCommandWrapper.java @@ -35,6 +35,7 @@ public Answer execute(final SetHostParamsCommand command, final LibvirtComputing final Map params = command.getParams(); boolean success = libvirtComputingResource.getVirtRouterResource().configureHostParams(params); + success = success && libvirtComputingResource.configureHostParams(params); if (!success) { return new Answer(command, false, "Failed to set host parameters"); From e86020335ef5a85d928a7939c4b6a52545808be7 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 31 Jan 2020 14:08:58 +0000 Subject: [PATCH 2/5] kvm: bug fix in config host param --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 9f56e1c0be86..db88b527aadd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1104,9 +1104,9 @@ public boolean configureHostParams(final Map params) { // Save configurations in agent.properties PropertiesStorage storage = new PropertiesStorage(); storage.configure("Storage", new HashMap()); - if (params.get("router.aggregation.command.each.timeout") == null) { + if (params.get("router.aggregation.command.each.timeout") != null) { String value = (String)params.get("router.aggregation.command.each.timeout"); - Integer intValue = NumbersUtil.parseInt(value, 10); + Integer intValue = NumbersUtil.parseInt(value, 600); storage.persist("router.aggregation.command.each.timeout", String.valueOf(intValue)); } From 8891da3935948ae77b07133d4f471cac7a4d74fb Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 31 Jan 2020 11:22:11 +0000 Subject: [PATCH 3/5] kvm: refactoring of propagating host params --- .../java/com/cloud/agent/AgentManager.java | 2 ++ .../cloud/agent/manager/AgentManagerImpl.java | 23 ++------------- .../cloudstack/agent/lb/IndirectAgentLB.java | 5 +++- .../ConfigurationManagerImpl.java | 28 +++++++++++++++++++ .../agent/lb/IndirectAgentLBServiceImpl.java | 27 ++---------------- 5 files changed, 39 insertions(+), 46 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index c51970c85f76..1d8e5faf9d4a 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -152,4 +152,6 @@ public enum TapAgentsAction { void notifyMonitorsOfHostAboutToBeRemoved(long hostId); void notifyMonitorsOfRemovedHost(long hostId, long clusterId); + + void propagateChangeToAgents(); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index dc9f93006f0e..4524f8334f8e 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -45,8 +45,6 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; @@ -85,7 +83,6 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; -import com.cloud.event.EventTypes; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.ConnectionException; import com.cloud.exception.OperationTimedoutException; @@ -190,8 +187,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl ResourceManager _resourceMgr; @Inject ManagementServiceConfiguration mgmtServiceConf; - @Inject - MessageBus messageBus; protected final ConfigKey Workers = new ConfigKey("Advanced", Integer.class, "workers", "5", "Number of worker threads handling remote agent connections.", false); @@ -242,8 +237,6 @@ public boolean configure(final String name, final Map params) th _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor")); - initMessageBusListener(); - return true; } @@ -1828,24 +1821,12 @@ private void sendCommandToAgents(Map> hostsPerZone, Map> hostsPerZone = getHostsPerZone(); Map params = new HashMap(); params.put("router.aggregation.command.each.timeout", _configDao.getValue("router.aggregation.command.each.timeout")); sendCommandToAgents(hostsPerZone, params); } - - private void initMessageBusListener() { - messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() { - @Override - public void onPublishMessage(String serderAddress, String subject, Object args) { - String globalSettingUpdated = (String)args; - if (globalSettingUpdated.equals("router.aggregation.command.each.timeout")) { - propagateChangeToAgents(); - } - } - }); - } - } diff --git a/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java b/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java index 627a5ee5f50c..464c489af848 100644 --- a/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java +++ b/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java @@ -50,4 +50,7 @@ public interface IndirectAgentLB { * @return returns interval in seconds */ Long getLBPreferredHostCheckInterval(Long clusterId); -} \ No newline at end of file + + void propagateMSListToAgents(); + +} diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 1f8655d40c93..be01b9f83259 100755 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.configuration; +import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.NetworkOfferingJoinDao; @@ -161,6 +162,8 @@ import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.agent.lb.IndirectAgentLB; +import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd; import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd; @@ -187,6 +190,7 @@ import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd; import org.apache.cloudstack.api.command.admin.zone.UpdateZoneCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd; +import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -198,6 +202,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpDao; @@ -372,6 +377,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati ImageStoreDetailsDao _imageStoreDetailsDao; @Inject MessageBus messageBus; + @Inject + AgentManager _agentManager; + @Inject + IndirectAgentLB _indirectAgentLB; // FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao? @@ -404,6 +413,7 @@ public boolean configure(final String name, final Map params) th populateConfigValuesForValidationSet(); weightBasedParametersForValidation(); overProvisioningFactorsForValidation(); + initMessageBusListener(); return true; } @@ -461,6 +471,24 @@ private void overProvisioningFactorsForValidation() { overprovisioningFactorsForValidation.add(CapacityManager.StorageOverprovisioningFactor.key()); } + private void initMessageBusListener() { + messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() { + @Override + public void onPublishMessage(String serderAddress, String subject, Object args) { + String globalSettingUpdated = (String) args; + if (Strings.isNullOrEmpty(globalSettingUpdated)) { + return; + } + if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) || + globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) { + _indirectAgentLB.propagateMSListToAgents(); + } else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString())) { + _agentManager.propagateChangeToAgents(); + } + } + }); + } + @Override public boolean start() { diff --git a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java index 0e9b32cb5dd7..8742e2dfe6f3 100644 --- a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java @@ -34,13 +34,10 @@ import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; -import com.cloud.event.EventTypes; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -68,8 +65,6 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement @Inject private HostDao hostDao; @Inject - private MessageBus messageBus; - @Inject private AgentManager agentManager; ////////////////////////////////////////////////////// @@ -208,7 +203,8 @@ private org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm getAgentMSLBAlgo /////////////// Agent MSLB Configuration /////////////////// //////////////////////////////////////////////////////////// - private void propagateMSListToAgents() { + @Override + public void propagateMSListToAgents() { LOG.debug("Propagating management server list update to agents"); final String lbAlgorithm = getLBAlgorithmName(); final Map> dcOrderedHostsMap = new HashMap<>(); @@ -227,22 +223,6 @@ private void propagateMSListToAgents() { } } - private void configureMessageBusListener() { - messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() { - @Override - public void onPublishMessage(final String senderAddress, String subject, Object args) { - final String globalSettingUpdated = (String) args; - if (Strings.isNullOrEmpty(globalSettingUpdated)) { - return; - } - if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) || - globalSettingUpdated.equals(IndirectAgentLBAlgorithm.key())) { - propagateMSListToAgents(); - } - } - }); - } - private void configureAlgorithmMap() { final List algorithms = new ArrayList<>(); algorithms.add(new IndirectAgentLBStaticAlgorithm()); @@ -258,7 +238,6 @@ private void configureAlgorithmMap() { public boolean configure(final String name, final Map params) throws ConfigurationException { super.configure(name, params); configureAlgorithmMap(); - configureMessageBusListener(); return true; } @@ -274,4 +253,4 @@ public ConfigKey[] getConfigKeys() { IndirectAgentLBCheckInterval }; } -} \ No newline at end of file +} From 3d97af5438288af1c4da0bdc03f6593561191b1d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 3 Feb 2020 15:06:30 +0000 Subject: [PATCH 4/5] test: fix build error in server/test --- server/src/test/resources/createNetworkOffering.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/resources/createNetworkOffering.xml b/server/src/test/resources/createNetworkOffering.xml index 6aac8e18aa86..32596fc03a27 100644 --- a/server/src/test/resources/createNetworkOffering.xml +++ b/server/src/test/resources/createNetworkOffering.xml @@ -58,4 +58,5 @@ + From f8ce84b5de085bcf5a41a3b60d106ea431dd0637 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 7 Feb 2020 22:20:32 +0000 Subject: [PATCH 5/5] server: fix test error in IndirectAgentLBServiceImplTest.java --- .../cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java index 79b5421f9af7..98c2af5b7764 100644 --- a/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java @@ -95,7 +95,6 @@ private void configureMocks() throws NoSuchFieldException, IllegalAccessExceptio id++; } addField(agentMSLB, "hostDao", hostDao); - addField(agentMSLB, "messageBus", messageBus); addField(agentMSLB, "agentManager", agentManager); when(hostDao.listAll()).thenReturn(Arrays.asList(host4, host2, host1, host3)); @@ -205,4 +204,4 @@ public void testGetHostsPerZoneNullHosts() { when(hostDao.listAll()).thenReturn(null); Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_2_ID).size() == 0); } -} \ No newline at end of file +}