From 3bfbbc8a12f78898ec7b61ef896171dabb08029e Mon Sep 17 00:00:00 2001 From: YeoCheon Yun Date: Wed, 20 Oct 2021 17:05:52 +0900 Subject: [PATCH 01/38] Replace multiple + and / symbols by - and _ to have URL-safe base64 going to the API --- ui/src/views/infra/AddPrimaryStorage.vue | 4 ++-- ui/src/views/infra/zone/ZoneWizardLaunchZone.vue | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/src/views/infra/AddPrimaryStorage.vue b/ui/src/views/infra/AddPrimaryStorage.vue index 8703ddbfd2a3..df674baf8913 100644 --- a/ui/src/views/infra/AddPrimaryStorage.vue +++ b/ui/src/views/infra/AddPrimaryStorage.vue @@ -547,8 +547,8 @@ export default { /* Replace the + and / symbols by - and _ to have URL-safe base64 going to the API It's hacky, but otherwise we'll confuse java.net.URI which splits the incoming URI */ - secret = secret.replace('+', '-') - secret = secret.replace('/', '_') + secret = secret.replace(/\+/g, '-') + secret = secret.replace(/\//g, '_') if (id !== null && secret !== null) { monitor = id + ':' + secret + '@' + monitor } diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index 7191f1935fdd..7ecbfaddb806 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -2091,8 +2091,8 @@ export default { }, rbdURL (monitor, pool, id, secret) { let url - secret = secret.replace('+', '-') - secret = secret.replace('/', '_') + secret = secret.replace(/\+/g, '-') + secret = secret.replace(/\//g, '_') if (id != null && secret != null) { monitor = id + ':' + secret + '@' + monitor } From c4f9c6e71a706c6d75d623cce9e9f01b4b66f665 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Tue, 28 Sep 2021 17:08:56 +0900 Subject: [PATCH 02/38] =?UTF-8?q?rbd=20multi=20monitor=20=EC=A7=80?= =?UTF-8?q?=EC=9B=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rbd multi monitor 지원 --- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index f39b17076e7a..a44516344320 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -244,7 +244,24 @@ public DataStore initialize(Map dsInfos) { port = 0; } parameters.setType(StoragePoolType.RBD); - parameters.setHost(storageHost); + if (storageHost == null) { + String urlSplit = url.substring(url.lastIndexOf("@")+1); + String urlHost = urlSplit.substring(0, urlSplit.indexOf("/")); + if (urlHost != null) { + String[] array = urlHost.split(","); + if (array != null) { + if (array.length < 6) { + parameters.setHost(urlHost); + } else { + throw new InvalidParameterValueException("RADOS monitor can support up to 5."); + } + } else { + throw new InvalidParameterValueException("Please input a valid RADOS monitor."); + } + } + } else { + parameters.setHost(storageHost); + } parameters.setPort(port); parameters.setPath(hostPath.replaceFirst("/", "")); parameters.setUserInfo(userInfo); From 2026348a299961b4d55a76de5f53a68858097a30 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 29 Sep 2021 10:42:44 +0900 Subject: [PATCH 03/38] =?UTF-8?q?comma=EB=A1=9C=20=EB=84=A3=EC=9D=84=20?= =?UTF-8?q?=EA=B2=BD=EC=9A=B0=20userInfo=20=EB=8F=84=20null=20=EB=A1=9C=20?= =?UTF-8?q?=EC=B6=9C=EB=A0=A5=EB=90=98=EB=8A=94=20=ED=98=84=EC=83=81=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit comma로 넣을 경우 userInfo 도 null 로 출력되는 현상 수정 --- .../lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index a44516344320..1bea1027b0a2 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -247,11 +247,14 @@ public DataStore initialize(Map dsInfos) { if (storageHost == null) { String urlSplit = url.substring(url.lastIndexOf("@")+1); String urlHost = urlSplit.substring(0, urlSplit.indexOf("/")); + String urlSplit2 = url.substring(url.lastIndexOf("//")+1); + String urlUserInfo = urlSplit2.substring(0, urlSplit2.indexOf("@")); if (urlHost != null) { String[] array = urlHost.split(","); if (array != null) { if (array.length < 6) { parameters.setHost(urlHost); + parameters.setUserInfo(urlUserInfo); } else { throw new InvalidParameterValueException("RADOS monitor can support up to 5."); } @@ -261,10 +264,10 @@ public DataStore initialize(Map dsInfos) { } } else { parameters.setHost(storageHost); + parameters.setUserInfo(userInfo); } parameters.setPort(port); parameters.setPath(hostPath.replaceFirst("/", "")); - parameters.setUserInfo(userInfo); } else if (scheme.equalsIgnoreCase("PreSetup")) { if (HypervisorType.VMware.equals(hypervisorType)) { validateVcenterDetails(zoneId, podId, clusterId,storageHost); From a2a12cd086c28f98e7261ef42910dcfa7630231f Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Wed, 29 Sep 2021 14:07:58 +0900 Subject: [PATCH 04/38] =?UTF-8?q?rbd=EC=9D=BC=20=EB=95=8C=20uuid=20?= =?UTF-8?q?=EC=97=90=EB=9F=AC=EB=A1=9C=20=EC=83=9D=EC=84=B1=EB=A1=9C?= =?UTF-8?q?=EC=A7=81=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rbd일 때 uuid 에러로 생성로직 변경 --- .../lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 1bea1027b0a2..edadff7dfda7 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -247,7 +247,7 @@ public DataStore initialize(Map dsInfos) { if (storageHost == null) { String urlSplit = url.substring(url.lastIndexOf("@")+1); String urlHost = urlSplit.substring(0, urlSplit.indexOf("/")); - String urlSplit2 = url.substring(url.lastIndexOf("//")+1); + String urlSplit2 = url.substring(url.lastIndexOf("//")+2); String urlUserInfo = urlSplit2.substring(0, urlSplit2.indexOf("@")); if (urlHost != null) { String[] array = urlHost.split(","); @@ -359,6 +359,8 @@ public DataStore initialize(Map dsInfos) { uuid = UUID.randomUUID().toString(); } else if ("PreSetup".equalsIgnoreCase(scheme) && !HypervisorType.VMware.equals(hypervisorType)) { uuid = hostPath.replace("/", ""); + } else if (scheme.equalsIgnoreCase("rbd")) { + uuid = UUID.nameUUIDFromBytes((parameters.getHost() + hostPath).getBytes()).toString(); } else { uuid = UUID.nameUUIDFromBytes((storageHost + hostPath).getBytes()).toString(); } From 85096797a1c1b63e0049038f9b29f512cdeafa7d Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Thu, 30 Sep 2021 14:52:49 +0900 Subject: [PATCH 05/38] =?UTF-8?q?multi=20monitor=EC=9D=B8=20=EA=B2=BD?= =?UTF-8?q?=EC=9A=B0=20=EB=A1=9C=EC=A7=81=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit multi monitor인 경우 로직 변경 --- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index edadff7dfda7..d4cc3c0e6a58 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -139,7 +139,13 @@ public DataStore initialize(Map dsInfos) { PrimaryDataStoreParameters parameters = new PrimaryDataStoreParameters(); URI uri = null; + boolean multi = false; try { + String urlType = url.substring(0, 3); + if (urlType.equals("rbd") && url.contains(",")) { + multi = true; + url = url.replaceAll(",", "/"); + } uri = new URI(UriUtils.encodeURIComponent(url)); if (uri.getScheme() == null) { throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); @@ -162,10 +168,18 @@ public DataStore initialize(Map dsInfos) { throw new InvalidParameterValueException("host or path is null, should be sharedmountpoint://localhost/path"); } } else if (uri.getScheme().equalsIgnoreCase("rbd")) { + String uriHost = uri.getHost(); String uriPath = uri.getPath(); if (uriPath == null) { throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool"); } + if (multi) { + String multiHost = uriHost + (uriPath.substring(0, uriPath.lastIndexOf("/")).replaceAll("/", ",")); + String[] hostArr = multiHost.split(","); + if (hostArr.length > 5) { + throw new InvalidParameterValueException("RADOS monitor can support up to 5."); + } + } } else if (uri.getScheme().equalsIgnoreCase("gluster")) { String uriHost = uri.getHost(); String uriPath = uri.getPath(); @@ -243,29 +257,13 @@ public DataStore initialize(Map dsInfos) { if (port == -1) { port = 0; } - parameters.setType(StoragePoolType.RBD); - if (storageHost == null) { - String urlSplit = url.substring(url.lastIndexOf("@")+1); - String urlHost = urlSplit.substring(0, urlSplit.indexOf("/")); - String urlSplit2 = url.substring(url.lastIndexOf("//")+2); - String urlUserInfo = urlSplit2.substring(0, urlSplit2.indexOf("@")); - if (urlHost != null) { - String[] array = urlHost.split(","); - if (array != null) { - if (array.length < 6) { - parameters.setHost(urlHost); - parameters.setUserInfo(urlUserInfo); - } else { - throw new InvalidParameterValueException("RADOS monitor can support up to 5."); - } - } else { - throw new InvalidParameterValueException("Please input a valid RADOS monitor."); - } - } - } else { - parameters.setHost(storageHost); - parameters.setUserInfo(userInfo); + if (multi) { + storageHost = storageHost + (hostPath.substring(0, hostPath.lastIndexOf("/")).replaceAll("/", ",")); + hostPath = hostPath.substring(hostPath.lastIndexOf("/")+1); } + parameters.setType(StoragePoolType.RBD); + parameters.setHost(storageHost); + parameters.setUserInfo(userInfo); parameters.setPort(port); parameters.setPath(hostPath.replaceFirst("/", "")); } else if (scheme.equalsIgnoreCase("PreSetup")) { @@ -359,8 +357,6 @@ public DataStore initialize(Map dsInfos) { uuid = UUID.randomUUID().toString(); } else if ("PreSetup".equalsIgnoreCase(scheme) && !HypervisorType.VMware.equals(hypervisorType)) { uuid = hostPath.replace("/", ""); - } else if (scheme.equalsIgnoreCase("rbd")) { - uuid = UUID.nameUUIDFromBytes((parameters.getHost() + hostPath).getBytes()).toString(); } else { uuid = UUID.nameUUIDFromBytes((storageHost + hostPath).getBytes()).toString(); } From 3f3ccdb352664da6c44ead709f77557af9795e46 Mon Sep 17 00:00:00 2001 From: Dajeong-Park Date: Thu, 30 Sep 2021 14:56:39 +0900 Subject: [PATCH 06/38] =?UTF-8?q?=EC=BD=94=EB=93=9C=20=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 코드 정리 --- .../lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index d4cc3c0e6a58..a08e4c519ff0 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -263,9 +263,9 @@ public DataStore initialize(Map dsInfos) { } parameters.setType(StoragePoolType.RBD); parameters.setHost(storageHost); - parameters.setUserInfo(userInfo); parameters.setPort(port); parameters.setPath(hostPath.replaceFirst("/", "")); + parameters.setUserInfo(userInfo); } else if (scheme.equalsIgnoreCase("PreSetup")) { if (HypervisorType.VMware.equals(hypervisorType)) { validateVcenterDetails(zoneId, podId, clusterId,storageHost); From 45c761ce1551a2c1b3f9f7cd9dde77d810bd259d Mon Sep 17 00:00:00 2001 From: sjyu Date: Tue, 7 Dec 2021 13:37:59 +0900 Subject: [PATCH 07/38] =?UTF-8?q?Host=20HA=20:=20RBD=20Primary=20Storage?= =?UTF-8?q?=EC=97=90=20=EB=8C=80=ED=95=9C=20Host=20HA=20=EA=B8=B0=EB=8A=A5?= =?UTF-8?q?=20=EC=A0=9C=EA=B3=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Primary Storage가 RBD인 경우에 Host HA가 동작하도록 기능개선 --- .../hypervisor/kvm/resource/KVMHABase.java | 10 +- .../hypervisor/kvm/resource/KVMHAChecker.java | 31 +++- .../hypervisor/kvm/resource/KVMHAMonitor.java | 38 ++++- .../kvm/resource/KVMHAVMActivityChecker.java | 36 ++++- .../resource/LibvirtComputingResource.java | 20 ++- ...VMActivityOnStoragePoolCommandWrapper.java | 7 +- .../kvm/storage/KVMStoragePoolManager.java | 5 +- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh | 147 +++++++++++++++++ .../vm/hypervisor/kvm/kvmvmactivity_rbd.sh | 153 ++++++++++++++++++ 9 files changed, 429 insertions(+), 18 deletions(-) create mode 100755 scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh create mode 100755 scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index fd1122f07ad3..395822380c65 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -24,6 +24,7 @@ import org.libvirt.StoragePoolInfo; import org.libvirt.StoragePoolInfo.StoragePoolState; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.OutputInterpreter.AllLinesParser; import com.cloud.utils.script.Script; @@ -32,6 +33,7 @@ public class KVMHABase { private static final Logger s_logger = Logger.getLogger(KVMHABase.class); private long _timeout = 60000; /* 1 minutes */ protected static String s_heartBeatPath; + protected static String s_heartBeatPathRbd; protected long _heartBeatUpdateTimeout = 60000; protected long _heartBeatUpdateFreq = 60000; protected long _heartBeatUpdateMaxTries = 5; @@ -47,13 +49,19 @@ public static class NfsStoragePool { String _poolMountSourcePath; String _mountDestPath; PoolType _type; + StoragePoolType _poolType; + String _poolAuthSecret; + String _poolSourceHost; - public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type) { + public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, StoragePoolType poolType, String poolAuthSecret, String poolSourceHost) { _poolUUID = poolUUID; _poolIp = poolIp; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; _type = type; + _poolType = poolType; + _poolAuthSecret = poolAuthSecret; + _poolSourceHost = poolSourceHost; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 5ceaef2bb197..0413394f8f9c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -22,8 +22,11 @@ import org.apache.log4j.Logger; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import java.net.InetAddress; +import java.net.UnknownHostException; public class KVMHAChecker extends KVMHABase implements Callable { private static final Logger s_logger = Logger.getLogger(KVMHAChecker.class); @@ -36,6 +39,21 @@ public KVMHAChecker(List pools, String host) { this.hostIp = host; } + public static String getIpAddress(String sourceHost) { + try { + String[] hostArr = sourceHost.split(","); + String sourceHostIP = ""; + for (String host : hostArr) { + InetAddress addr = InetAddress.getByName(host); + sourceHostIP += addr.getHostAddress() + ","; + } + return sourceHostIP; + } catch (UnknownHostException e) { + s_logger.debug("Failed to get connection: " + e.getMessage()); + return null; + } + } + /* * True means heartbeaing is on going, or we can't get it's status. False * means heartbeating is stopped definitely @@ -50,9 +68,16 @@ public Boolean checkingHeartBeat() { for (NfsStoragePool pool : nfsStoragePools) { Script cmd = new Script(s_heartBeatPath, heartBeatCheckerTimeout, s_logger); - cmd.add("-i", pool._poolIp); - cmd.add("-p", pool._poolMountSourcePath); - cmd.add("-m", pool._mountDestPath); + if (pool._poolType == StoragePoolType.NetworkFilesystem) { + cmd.add("-i", pool._poolIp); + cmd.add("-p", pool._poolMountSourcePath); + cmd.add("-m", pool._mountDestPath); + } else if (pool._poolType == StoragePoolType.RBD) { + cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); + cmd.add("-i", getIpAddress(pool._poolSourceHost)); + cmd.add("-p", pool._poolMountSourcePath); + cmd.add("-s", pool._poolAuthSecret); + } cmd.add("-h", hostIp); cmd.add("-r"); cmd.add("-t", String.valueOf(_heartBeatUpdateFreq / 1000)); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 022b48c929f9..165d29dd60cd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -18,6 +18,7 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.Script; import org.apache.log4j.Logger; import org.libvirt.Connect; @@ -31,6 +32,8 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.net.InetAddress; +import java.net.UnknownHostException; public class KVMHAMonitor extends KVMHABase implements Runnable { @@ -40,12 +43,13 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private final String hostPrivateIp; - public KVMHAMonitor(NfsStoragePool pool, String host, String scriptPath) { + public KVMHAMonitor(NfsStoragePool pool, String host, String scriptPath, String scriptPathRbd) { if (pool != null) { storagePool.put(pool._poolUUID, pool); } hostPrivateIp = host; configureHeartBeatPath(scriptPath); + configureHeartBeatPathRbd(scriptPathRbd); _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); @@ -55,6 +59,10 @@ private static synchronized void configureHeartBeatPath(String scriptPath) { KVMHABase.s_heartBeatPath = scriptPath; } + private static synchronized void configureHeartBeatPathRbd(String scriptPathRbd) { + KVMHABase.s_heartBeatPathRbd = scriptPathRbd; + } + public void addStoragePool(NfsStoragePool pool) { synchronized (storagePool) { storagePool.put(pool._poolUUID, pool); @@ -83,6 +91,21 @@ public NfsStoragePool getStoragePool(String uuid) { } } + public static String getIpAddress(String sourceHost) { + try { + String[] hostArr = sourceHost.split(","); + String sourceHostIP = ""; + for (String host : hostArr) { + InetAddress addr = InetAddress.getByName(host); + sourceHostIP += addr.getHostAddress() + ","; + } + return sourceHostIP; + } catch (UnknownHostException e) { + s_logger.debug("Failed to get connection: " + e.getMessage()); + return null; + } + } + protected void runHeartBeat() { synchronized (storagePool) { Set removedPools = new HashSet<>(); @@ -154,9 +177,16 @@ protected void runHeartBeat() { private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { Script cmd = new Script(s_heartBeatPath, _heartBeatUpdateTimeout, s_logger); - cmd.add("-i", primaryStoragePool._poolIp); - cmd.add("-p", primaryStoragePool._poolMountSourcePath); - cmd.add("-m", primaryStoragePool._mountDestPath); + if (primaryStoragePool._poolType == StoragePoolType.NetworkFilesystem) { + cmd.add("-i", primaryStoragePool._poolIp); + cmd.add("-p", primaryStoragePool._poolMountSourcePath); + cmd.add("-m", primaryStoragePool._mountDestPath); + } else if (primaryStoragePool._poolType == StoragePoolType.RBD) { + cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); + cmd.add("-i", getIpAddress(primaryStoragePool._poolSourceHost)); + cmd.add("-p", primaryStoragePool._poolMountSourcePath); + cmd.add("-s", primaryStoragePool._poolAuthSecret); + } if (hostValidation) { cmd.add("-h", hostPrivateIp); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 758edd2a24b0..3b2433661727 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -18,10 +18,13 @@ import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; +import com.cloud.storage.Storage.StoragePoolType; import org.apache.log4j.Logger; import org.joda.time.Duration; import java.util.concurrent.Callable; +import java.net.InetAddress; +import java.net.UnknownHostException; public class KVMHAVMActivityChecker extends KVMHABase implements Callable { private static final Logger LOG = Logger.getLogger(KVMHAVMActivityChecker.class); @@ -32,21 +35,44 @@ public class KVMHAVMActivityChecker extends KVMHABase implements Callable params) th throw new ConfigurationException("Unable to find kvmheartbeat.sh"); } + _heartBeatPathRbd = Script.findScript(kvmScriptsDir, "kvmheartbeat_rbd.sh"); + if (_heartBeatPathRbd == null) { + throw new ConfigurationException("Unable to find kvmheartbeat_rbd.sh"); + } + _createvmPath = Script.findScript(storageScriptsDir, "createvm.sh"); if (_createvmPath == null) { throw new ConfigurationException("Unable to find the createvm.sh"); @@ -892,6 +903,11 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find kvmvmactivity.sh"); } + _vmActivityCheckPathRbd = Script.findScript(kvmScriptsDir, "kvmvmactivity_rbd.sh"); + if (_vmActivityCheckPathRbd == null) { + throw new ConfigurationException("Unable to find kvmvmactivity_rbd.sh"); + } + _createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (_createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); @@ -1163,7 +1179,7 @@ public boolean configure(final String name, final Map params) th final String[] info = NetUtils.getNetworkParams(_privateNic); - _monitor = new KVMHAMonitor(null, info[0], _heartBeatPath); + _monitor = new KVMHAMonitor(null, info[0], _heartBeatPath, _heartBeatPathRbd); final Thread ha = new Thread(_monitor); ha.start(); @@ -3752,7 +3768,7 @@ private DiskDef.DiskBus getGuestDiskModel(final String platformEmulator, boolean } else if (platformEmulator.startsWith("Other PV Virtio-SCSI")) { return DiskDef.DiskBus.SCSI; } else if (platformEmulator.contains("Ubuntu") || - StringUtils.startsWithAny(platformEmulator, + StringUtils.startsWithAny(platformEmulator, "Fedora", "CentOS", "Red Hat Enterprise Linux", "Debian GNU/Linux", "FreeBSD", "Oracle", "Other PV")) { return DiskDef.DiskBus.VIRTIO; } else if (isUefiEnabled && StringUtils.startsWithAny(platformEmulator, "Windows", "Other")) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index 9c899a040550..85ff7b420928 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -43,9 +43,12 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L final ExecutorService executors = Executors.newSingleThreadExecutor(); final KVMHAMonitor monitor = libvirtComputingResource.getMonitor(); final StorageFilerTO pool = command.getPool(); - if (Storage.StoragePoolType.NetworkFilesystem == pool.getType()){ + if (Storage.StoragePoolType.NetworkFilesystem == pool.getType() || Storage.StoragePoolType.RBD == pool.getType()){ final NfsStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost().getPrivateNetwork().getIp(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + String vmActivityCheckPath = ""; + if (Storage.StoragePoolType.NetworkFilesystem == pool.getType()) vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPath(); + else if (Storage.StoragePoolType.RBD == pool.getType()) vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPathRbd(); + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost().getPrivateNetwork().getIp(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds(), pool.getType()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 77579c4eca6b..0ab40f1ccd2c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -331,7 +331,10 @@ private synchronized KVMStoragePool createStoragePool(String name, String host, // LibvirtStorageAdaptor-specific statement if (type == StoragePoolType.NetworkFilesystem && primaryStorage) { - KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage); + KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.NetworkFilesystem, null, null); + _haMonitor.addStoragePool(nfspool); + } else if (type == StoragePoolType.RBD && primaryStorage) { + KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.RBD, pool.getAuthSecret(), pool.getSourceHost()); _haMonitor.addStoragePool(nfspool); } StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, primaryStorage); diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh new file mode 100755 index 000000000000..84124d6619b9 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +help() { + printf "Usage: $0 + -p rbd pool name + -s pool auth secret + -h host + -i source host ip + -r write/read hb log + -c cleanup + -t interval between read hb log\n" + exit 1 +} +#set -x +PoolName= +PoolAuthSecret= +HostIP= +SourceHostIP= +interval= +rflag=0 +cflag=0 + +while getopts 'p:s:h:i:t:rc' OPTION +do + case $OPTION in + p) + PoolName="$OPTARG" + ;; + s) + PoolAuthSecret="$OPTARG" + ;; + h) + HostIP="$OPTARG" + ;; + i) + SourceHostIP="$OPTARG" + ;; + t) + interval="$OPTARG" + ;; + r) + rflag=1 + ;; + c) + cflag=1 + ;; + *) + help + ;; + esac +done + +if [ -z "$PoolName" ]; then + exit 2 +fi + +keyringFile="/etc/ceph/keyring.bin" +confFile="/etc/ceph/ceph.conf" + +create_cephConf() { +#Creating Ceph keyring and conf for executing rados commands + if [ ! -f $keyringFile ]; then + echo -e "[client.admin]\n key=$PoolAuthSecret" > $keyringFile + fi + + if [ ! -f $confFile ]; then + confContents="[global]\n mon host = " + for ip in $(echo $SourceHostIP | sed 's/,/ /g'); do + confContents+="[v2:${ip}:3300/0,v1:${ip}:6789/0], " + done + echo -e "$confContents" | sed 's/, $//' > $confFile + fi +} + +delete_cephConf() { +#Deleting Ceph keyring + if [ -f $keyringFile ]; then + rm -rf $keyringFile + fi +} + +write_hbLog() { +#write the heart beat log + timestamp=$(date +%s) + obj=$(rados -p $PoolName ls | grep hb_$HostIP) + if [ $? -gt 0 ]; then + rados -p $PoolName create hb_$HostIP + fi + echo $timestamp | rados -p $PoolName put hb_$HostIP - + if [ $? -gt 0 ]; then + printf "Failed to create rbd file" + return 2 + fi + return 0 +} + +check_hbLog() { +#check the heart beat log + now=$(date +%s) + #$(rados -p $PoolName get hb_$HostIP hb-$HostIP | cat hb-$HostIP) + hb=$(rados -p $PoolName get hb_$HostIP -) + diff=`expr $now - $hb` + if [ $diff -gt $interval ]; then + return $diff + fi + return 0 +} + +if [ "$rflag" == "1" ]; then + create_cephConf + check_hbLog + diff=$? + if [ $diff == 0 ]; then + echo "=====> ALIVE <=====" + else + echo "=====> Considering host as DEAD because last write on [RBD pool] was [$diff] seconds ago, but the max interval is [$interval] <======" + fi + delete_cephConf + exit 0 +elif [ "$cflag" == "1" ]; then + /usr/bin/logger -t heartbeat "kvmheartbeat_rbd.sh will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? +else + create_cephConf + write_hbLog + delete_cephConf + exit 0 +fi diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh new file mode 100755 index 000000000000..2cc031c76ec7 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +help() { + printf "Usage: $0 + -p rbd pool name + -s pool auth secret + -h host + -i source host ip + -u volume uuid list + -t time on ms + -d suspect time\n" + exit 1 +} +#set -x +PoolName= +PoolAuthSecret= +HostIP= +SourceHostIP= +UUIDList= +MSTime= +SuspectTime= + +while getopts 'p:s:h:i:u:t:d:' OPTION +do + case $OPTION in + p) + PoolName="$OPTARG" + ;; + s) + PoolAuthSecret="$OPTARG" + ;; + h) + HostIP="$OPTARG" + ;; + i) + SourceHostIP="$OPTARG" + ;; + u) + UUIDList="$OPTARG" + ;; + t) + MSTime="$OPTARG" + ;; + d) + SuspectTime="$OPTARG" + ;; + *) + help + ;; + esac +done + +if [ -z "$PoolName" ]; then + exit 2 +fi + +if [ -z "$SuspectTime" ]; then + exit 2 +fi + +#Creating Ceph keyring and conf for executing rados commands +keyringFile="/etc/ceph/keyring.bin" +confFile="/etc/ceph/ceph.conf" + +if [ ! -f $keyringFile ]; then + echo -e "[client.admin]\n key=$PoolAuthSecret" > $keyringFile +fi + +if [ ! -f $confFile ]; then + confContents="[global]\n mon host = " + for ip in $(echo $SourceHostIP | sed 's/,/ /g'); do + confContents+="[v2:${ip}:3300/0,v1:${ip}:6789/0], " + done + echo -e "$confContents" | sed 's/, $//' > $confFile +fi + +# First check: heartbeat file +now=$(date +%s) +#hb=$(rados -p $PoolName get hb_$HostIP hb_$HostIP | cat hb_$HostIP) +hb=$(rados -p $PoolName get hb_$HostIP -) +diff=$(expr $now - $hb) +if [ $diff -lt 61 ]; then + echo "=====> ALIVE <=====" + exit 0 +fi + +if [ -z "$UUIDList" ]; then + echo "=====> Considering host as DEAD due to empty UUIDList <======" + exit 0 +fi + +# Second check: disk activity check +lastestUUIDList= +for UUID in $(echo $UUIDList | sed 's/,/ /g'); do + time=$(rbd info $UUID | grep modify_timestamp) + time=${time#*modify_timestamp: } + time=$(date -d "$time" +%s) + lastestUUIDList+="${time}\n" +done + +latestUpdateTime=$(echo -e $lastestUUIDList 2> /dev/null | sort -nr | head -1) +obj=$(rados -p $PoolName ls | grep ac_$HostIP) +if [ $? -gt 0 ]; then + rados -p $PoolName create ac_$HostIP + echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac_$HostIP - + if [[ $latestUpdateTime -gt $SuspectTime ]]; then + echo "=====> ALIVE <=====" + else + echo "=====> Considering host as DEAD due to file [RBD pool] does not exists and condition [latestUpdateTime -gt SuspectTime] has not been satisfied. <======" + fi +else + acTime=$(rados -p $PoolName get ac_$HostIP -) + arrTime=(${acTime//:/ }) + lastSuspectTime=${arrTime[0]} + lastUpdateTime=${arrTime[1]} + echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac_$HostIP - + suspectTimeDiff=$(expr $SuspectTime - $lastSuspectTime) + if [ $suspectTimeDiff -lt 0 ]; then + if [[ $latestUpdateTime -gt $SuspectTime ]]; then + echo "=====> ALIVE <=====" + else + echo "=====> Considering host as DEAD due to file [RBD pool] exist, condition [suspectTimeDiff -lt 0] was satisfied and [latestUpdateTime -gt SuspectTime] has not been satisfied. <======" + fi + else + if [[ $latestUpdateTime -gt $lastUpdateTime ]]; then + echo "=====> ALIVE <=====" + else + echo "=====> Considering host as DEAD due to file [RBD pool] exist and conditions [suspectTimeDiff -lt 0] and [latestUpdateTime -gt SuspectTime] have not been satisfied. <======" + fi + fi +fi + +#Deleting Ceph keyring +if [ -f $keyringFile ]; then + rm -rf $keyringFile +fi + +exit 0 \ No newline at end of file From eb953b5cad5e4436913cf0979e47406c449d4097 Mon Sep 17 00:00:00 2001 From: sjyu Date: Thu, 9 Dec 2021 10:11:28 +0900 Subject: [PATCH 08/38] =?UTF-8?q?Host=20HA=20:=20RBD=20Primary=20Storage?= =?UTF-8?q?=EC=97=90=20=EB=8C=80=ED=95=9C=20Host=20HA=20=EA=B8=B0=EB=8A=A5?= =?UTF-8?q?=20=EC=A0=9C=EA=B3=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nfsstoragepool, rbdstoragepool 분리 --- .../hypervisor/kvm/resource/KVMHABase.java | 20 +- .../hypervisor/kvm/resource/KVMHAChecker.java | 41 ++- .../hypervisor/kvm/resource/KVMHAMonitor.java | 250 ++++++++++++------ .../kvm/resource/KVMHAVMActivityChecker.java | 15 +- .../resource/LibvirtComputingResource.java | 4 +- .../LibvirtCheckOnHostCommandWrapper.java | 6 +- ...VMActivityOnStoragePoolCommandWrapper.java | 14 +- .../wrapper/LibvirtFenceCommandWrapper.java | 12 +- .../kvm/storage/KVMStoragePoolManager.java | 6 +- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh | 11 +- .../vm/hypervisor/kvm/kvmvmactivity_rbd.sh | 13 +- 11 files changed, 271 insertions(+), 121 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 395822380c65..37f485b8b7c2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -50,10 +50,28 @@ public static class NfsStoragePool { String _mountDestPath; PoolType _type; StoragePoolType _poolType; + + public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, StoragePoolType poolType) { + _poolUUID = poolUUID; + _poolIp = poolIp; + _poolMountSourcePath = poolSourcePath; + _mountDestPath = mountDestPath; + _type = type; + _poolType = poolType; + } + } + + public static class RbdStoragePool { + String _poolUUID; + String _poolIp; + String _poolMountSourcePath; + String _mountDestPath; + PoolType _type; + StoragePoolType _poolType; String _poolAuthSecret; String _poolSourceHost; - public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, StoragePoolType poolType, String poolAuthSecret, String poolSourceHost) { + public RbdStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, StoragePoolType poolType, String poolAuthSecret, String poolSourceHost) { _poolUUID = poolUUID; _poolIp = poolIp; _poolMountSourcePath = poolSourcePath; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 0413394f8f9c..89a864b650e7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -22,7 +22,6 @@ import org.apache.log4j.Logger; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; import java.net.InetAddress; @@ -31,11 +30,13 @@ public class KVMHAChecker extends KVMHABase implements Callable { private static final Logger s_logger = Logger.getLogger(KVMHAChecker.class); private List nfsStoragePools; + private List rbdStoragePools; private String hostIp; private long heartBeatCheckerTimeout = 360000; // 6 minutes - public KVMHAChecker(List pools, String host) { - this.nfsStoragePools = pools; + public KVMHAChecker(List nfspools, List rbdpools, String host) { + this.nfsStoragePools = nfspools; + this.rbdStoragePools = rbdpools; this.hostIp = host; } @@ -68,16 +69,32 @@ public Boolean checkingHeartBeat() { for (NfsStoragePool pool : nfsStoragePools) { Script cmd = new Script(s_heartBeatPath, heartBeatCheckerTimeout, s_logger); - if (pool._poolType == StoragePoolType.NetworkFilesystem) { - cmd.add("-i", pool._poolIp); - cmd.add("-p", pool._poolMountSourcePath); - cmd.add("-m", pool._mountDestPath); - } else if (pool._poolType == StoragePoolType.RBD) { - cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); - cmd.add("-i", getIpAddress(pool._poolSourceHost)); - cmd.add("-p", pool._poolMountSourcePath); - cmd.add("-s", pool._poolAuthSecret); + cmd.add("-i", pool._poolIp); + cmd.add("-p", pool._poolMountSourcePath); + cmd.add("-m", pool._mountDestPath); + cmd.add("-h", hostIp); + cmd.add("-r"); + cmd.add("-t", String.valueOf(_heartBeatUpdateFreq / 1000)); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = cmd.execute(parser); + String parsedLine = parser.getLine(); + + s_logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, + pool._poolIp)); + + if (result == null && parsedLine.contains("DEAD")) { + s_logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned [%s]. [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), + result, parsedLine, hostIp)); + } else { + validResult = true; } + } + + for (RbdStoragePool pool : rbdStoragePools) { + Script cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); + cmd.add("-i", getIpAddress(pool._poolSourceHost)); + cmd.add("-p", pool._poolMountSourcePath); + cmd.add("-s", pool._poolAuthSecret); cmd.add("-h", hostIp); cmd.add("-r"); cmd.add("-t", String.valueOf(_heartBeatUpdateFreq / 1000)); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 165d29dd60cd..f0c79efb0de2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -18,7 +18,6 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.Script; import org.apache.log4j.Logger; import org.libvirt.Connect; @@ -38,56 +37,82 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private static final Logger s_logger = Logger.getLogger(KVMHAMonitor.class); - private final Map storagePool = new ConcurrentHashMap<>(); + private final Map nfsstoragePool = new ConcurrentHashMap<>(); + private final Map rbdstoragePool = new ConcurrentHashMap<>(); private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; - public KVMHAMonitor(NfsStoragePool pool, String host, String scriptPath, String scriptPathRbd) { + public KVMHAMonitor(NfsStoragePool pool, RbdStoragePool rbdpool, String host, String scriptPath, String scriptPathRbd) { if (pool != null) { - storagePool.put(pool._poolUUID, pool); + nfsstoragePool.put(pool._poolUUID, pool); + }else if (rbdpool != null) { + rbdstoragePool.put(rbdpool._poolUUID, rbdpool); } + configureHeartBeatPath(scriptPath, scriptPathRbd); hostPrivateIp = host; - configureHeartBeatPath(scriptPath); - configureHeartBeatPathRbd(scriptPathRbd); - _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } - private static synchronized void configureHeartBeatPath(String scriptPath) { + private static synchronized void configureHeartBeatPath(String scriptPath, String scriptPathRbd) { KVMHABase.s_heartBeatPath = scriptPath; - } - - private static synchronized void configureHeartBeatPathRbd(String scriptPathRbd) { KVMHABase.s_heartBeatPathRbd = scriptPathRbd; } public void addStoragePool(NfsStoragePool pool) { - synchronized (storagePool) { - storagePool.put(pool._poolUUID, pool); + synchronized (nfsstoragePool) { + nfsstoragePool.put(pool._poolUUID, pool); + } + } + + public void addStoragePool(RbdStoragePool pool) { + synchronized (rbdstoragePool) { + rbdstoragePool.put(pool._poolUUID, pool); } } public void removeStoragePool(String uuid) { - synchronized (storagePool) { - NfsStoragePool pool = storagePool.get(uuid); + synchronized (nfsstoragePool) { + NfsStoragePool pool = nfsstoragePool.get(uuid); + if (pool != null) { + Script.runSimpleBashScript("umount " + pool._mountDestPath); + nfsstoragePool.remove(uuid); + } + } + } + + public void removeRbdStoragePool(String uuid) { + synchronized (rbdstoragePool) { + RbdStoragePool pool = rbdstoragePool.get(uuid); if (pool != null) { Script.runSimpleBashScript("umount " + pool._mountDestPath); - storagePool.remove(uuid); + rbdstoragePool.remove(uuid); } } } public List getStoragePools() { - synchronized (storagePool) { - return new ArrayList<>(storagePool.values()); + synchronized (nfsstoragePool) { + return new ArrayList<>(nfsstoragePool.values()); + } + } + + public List getRbdStoragePools() { + synchronized (rbdstoragePool) { + return new ArrayList<>(rbdstoragePool.values()); } } public NfsStoragePool getStoragePool(String uuid) { - synchronized (storagePool) { - return storagePool.get(uuid); + synchronized (nfsstoragePool) { + return nfsstoragePool.get(uuid); + } + } + + public RbdStoragePool getRbdStoragePool(String uuid) { + synchronized (rbdstoragePool) { + return rbdstoragePool.get(uuid); } } @@ -107,87 +132,164 @@ public static String getIpAddress(String sourceHost) { } protected void runHeartBeat() { - synchronized (storagePool) { - Set removedPools = new HashSet<>(); - for (String uuid : storagePool.keySet()) { - NfsStoragePool primaryStoragePool = storagePool.get(uuid); - StoragePool storage; - try { - Connect conn = LibvirtConnection.getConnection(); - storage = conn.storagePoolLookupByUUIDString(uuid); - if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - if (storage == null) { - s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); - } else { - s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + if(nfsstoragePool != null && !nfsstoragePool.isEmpty()) { + synchronized (nfsstoragePool) { + Set removedPools = new HashSet<>(); + for (String uuid : nfsstoragePool.keySet()) { + NfsStoragePool primaryStoragePool = nfsstoragePool.get(uuid); + StoragePool storage; + try { + Connect conn = LibvirtConnection.getConnection(); + storage = conn.storagePoolLookupByUUIDString(uuid); + if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { + if (storage == null) { + s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + } else { + s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + } + + removedPools.add(uuid); + continue; + } + + s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + + } catch (LibvirtException e) { + s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + + if (e.toString().contains("pool not found")) { + s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + removedPools.add(uuid); + continue; } - removedPools.add(uuid); - continue; } - s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + String result = null; + for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + Script cmd = createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); + result = cmd.execute(); - } catch (LibvirtException e) { - s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); + + if (result != null) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); + try { + Thread.sleep(_heartBeatUpdateRetrySleep); + } catch (InterruptedException e) { + s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } + } else { + break; + } - if (e.toString().contains("pool not found")) { - s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); - removedPools.add(uuid); - continue; } + if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + Script cmd = createHeartBeatCommand(primaryStoragePool, null, false); + result = cmd.execute(); + } } - String result = null; - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { - Script cmd = createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - result = cmd.execute(); + if (!removedPools.isEmpty()) { + for (String uuid : removedPools) { + removeStoragePool(uuid); + } + } + } + } else if (rbdstoragePool != null && !rbdstoragePool.isEmpty()) { + synchronized (rbdstoragePool) { + Set removedPools = new HashSet<>(); + for (String uuid : rbdstoragePool.keySet()) { + RbdStoragePool primaryStoragePool = rbdstoragePool.get(uuid); + StoragePool storage; + try { + Connect conn = LibvirtConnection.getConnection(); + storage = conn.storagePoolLookupByUUIDString(uuid); + if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { + if (storage == null) { + s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + } else { + s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + } + + removedPools.add(uuid); + continue; + } + + s_logger.debug(String.format("Found RBD storage pool [%s] in libvirt, continuing.", uuid)); - s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); + } catch (LibvirtException e) { + s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); - if (result != null) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + if (e.toString().contains("pool not found")) { + s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + removedPools.add(uuid); + continue; } - } else { - break; + } - } + String result = null; + for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + Script cmd = createRbdHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); + result = cmd.execute(); + + s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); - if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); - Script cmd = createHeartBeatCommand(primaryStoragePool, null, false); - result = cmd.execute(); + if (result != null) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); + try { + Thread.sleep(_heartBeatUpdateRetrySleep); + } catch (InterruptedException e) { + s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } + } else { + break; + } + + } + + if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + Script cmd = createRbdHeartBeatCommand(primaryStoragePool, null, false); + result = cmd.execute(); + } } - } - if (!removedPools.isEmpty()) { - for (String uuid : removedPools) { - removeStoragePool(uuid); + if (!removedPools.isEmpty()) { + for (String uuid : removedPools) { + removeRbdStoragePool(uuid); + } } } } - } private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { Script cmd = new Script(s_heartBeatPath, _heartBeatUpdateTimeout, s_logger); - if (primaryStoragePool._poolType == StoragePoolType.NetworkFilesystem) { - cmd.add("-i", primaryStoragePool._poolIp); - cmd.add("-p", primaryStoragePool._poolMountSourcePath); - cmd.add("-m", primaryStoragePool._mountDestPath); - } else if (primaryStoragePool._poolType == StoragePoolType.RBD) { - cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); - cmd.add("-i", getIpAddress(primaryStoragePool._poolSourceHost)); - cmd.add("-p", primaryStoragePool._poolMountSourcePath); - cmd.add("-s", primaryStoragePool._poolAuthSecret); + cmd.add("-i", primaryStoragePool._poolIp); + cmd.add("-p", primaryStoragePool._poolMountSourcePath); + cmd.add("-m", primaryStoragePool._mountDestPath); + + if (hostValidation) { + cmd.add("-h", hostPrivateIp); } + if (!hostValidation) { + cmd.add("-c"); + } + + return cmd; + } + + private Script createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + Script cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); + cmd.add("-i", getIpAddress(primaryStoragePool._poolSourceHost)); + cmd.add("-p", primaryStoragePool._poolMountSourcePath); + cmd.add("-s", primaryStoragePool._poolAuthSecret); + if (hostValidation) { cmd.add("-h", hostPrivateIp); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 3b2433661727..207576a771f9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -30,6 +30,7 @@ public class KVMHAVMActivityChecker extends KVMHABase implements Callable params) th final String[] info = NetUtils.getNetworkParams(_privateNic); - _monitor = new KVMHAMonitor(null, info[0], _heartBeatPath, _heartBeatPathRbd); + _monitor = new KVMHAMonitor(null, null, info[0], _heartBeatPath, _heartBeatPathRbd); final Thread ha = new Thread(_monitor); ha.start(); @@ -3768,7 +3768,7 @@ private DiskDef.DiskBus getGuestDiskModel(final String platformEmulator, boolean } else if (platformEmulator.startsWith("Other PV Virtio-SCSI")) { return DiskDef.DiskBus.SCSI; } else if (platformEmulator.contains("Ubuntu") || - StringUtils.startsWithAny(platformEmulator, + StringUtils.startsWithAny(platformEmulator, "Fedora", "CentOS", "Red Hat Enterprise Linux", "Debian GNU/Linux", "FreeBSD", "Oracle", "Other PV")) { return DiskDef.DiskBus.VIRTIO; } else if (isUefiEnabled && StringUtils.startsWithAny(platformEmulator, "Windows", "Other")) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java index 651cdc9e888c..302913008182 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckOnHostCommandWrapper.java @@ -30,6 +30,7 @@ import com.cloud.agent.api.to.HostTO; import com.cloud.agent.api.to.NetworkTO; import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool; +import com.cloud.hypervisor.kvm.resource.KVMHABase.RbdStoragePool; import com.cloud.hypervisor.kvm.resource.KVMHAChecker; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; @@ -44,10 +45,11 @@ public Answer execute(final CheckOnHostCommand command, final LibvirtComputingRe final ExecutorService executors = Executors.newSingleThreadExecutor(); final KVMHAMonitor monitor = libvirtComputingResource.getMonitor(); - final List pools = monitor.getStoragePools(); + final List nfspools = monitor.getStoragePools(); + final List rbdpools = monitor.getRbdStoragePools(); final HostTO host = command.getHost(); final NetworkTO privateNetwork = host.getPrivateNetwork(); - final KVMHAChecker ha = new KVMHAChecker(pools, privateNetwork.getIp()); + final KVMHAChecker ha = new KVMHAChecker(nfspools, rbdpools, privateNetwork.getIp()); final Future future = executors.submit(ha); try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index 85ff7b420928..da4c90174dab 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -19,10 +19,16 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool; +import com.cloud.hypervisor.kvm.resource.KVMHABase.RbdStoragePool; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; import com.cloud.hypervisor.kvm.resource.KVMHAVMActivityChecker; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; @@ -30,11 +36,6 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - @ResourceWrapper(handles = CheckVMActivityOnStoragePoolCommand.class) public final class LibvirtCheckVMActivityOnStoragePoolCommandWrapper extends CommandWrapper { @@ -45,10 +46,11 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L final StorageFilerTO pool = command.getPool(); if (Storage.StoragePoolType.NetworkFilesystem == pool.getType() || Storage.StoragePoolType.RBD == pool.getType()){ final NfsStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); + final RbdStoragePool rbdpool = monitor.getRbdStoragePool(pool.getUuid()); String vmActivityCheckPath = ""; if (Storage.StoragePoolType.NetworkFilesystem == pool.getType()) vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPath(); else if (Storage.StoragePoolType.RBD == pool.getType()) vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPathRbd(); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost().getPrivateNetwork().getIp(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds(), pool.getType()); + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, rbdpool, command.getHost().getPrivateNetwork().getIp(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds(), pool.getType()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java index cb313dfa8998..4916c8c7c1fe 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java @@ -31,6 +31,7 @@ import com.cloud.agent.api.FenceAnswer; import com.cloud.agent.api.FenceCommand; import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool; +import com.cloud.hypervisor.kvm.resource.KVMHABase.RbdStoragePool; import com.cloud.hypervisor.kvm.resource.KVMHAChecker; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; @@ -47,20 +48,25 @@ public Answer execute(final FenceCommand command, final LibvirtComputingResource final ExecutorService executors = Executors.newSingleThreadExecutor(); final KVMHAMonitor monitor = libvirtComputingResource.getMonitor(); - final List pools = monitor.getStoragePools(); + final List nfspools = monitor.getStoragePools(); + final List rbdpools = monitor.getRbdStoragePools(); /** * We can only safely fence off hosts when we use NFS * On NFS primary storage pools hosts continuesly write * a heartbeat. Disable Fencing Off for hosts without NFS */ - if (pools.size() == 0) { + if (nfspools.size() == 0) { String logline = "No NFS storage pools found. No way to safely fence " + command.getVmName() + " on host " + command.getHostGuid(); s_logger.warn(logline); return new FenceAnswer(command, false, logline); + }else if (rbdpools.size() == 0) { + String logline = "No RBD storage pools found. No way to safely fence " + command.getVmName() + " on host " + command.getHostGuid(); + s_logger.warn(logline); + return new FenceAnswer(command, false, logline); } - final KVMHAChecker ha = new KVMHAChecker(pools, command.getHostIp()); + final KVMHAChecker ha = new KVMHAChecker(nfspools, rbdpools, command.getHostIp()); final Future future = executors.submit(ha); try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 0ab40f1ccd2c..1979ff750dce 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -331,11 +331,11 @@ private synchronized KVMStoragePool createStoragePool(String name, String host, // LibvirtStorageAdaptor-specific statement if (type == StoragePoolType.NetworkFilesystem && primaryStorage) { - KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.NetworkFilesystem, null, null); + KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.NetworkFilesystem); _haMonitor.addStoragePool(nfspool); } else if (type == StoragePoolType.RBD && primaryStorage) { - KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.RBD, pool.getAuthSecret(), pool.getSourceHost()); - _haMonitor.addStoragePool(nfspool); + KVMHABase.RbdStoragePool rbdpool = new KVMHABase.RbdStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.RBD, pool.getAuthSecret(), pool.getSourceHost()); + _haMonitor.addStoragePool(rbdpool); } StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, primaryStorage); addStoragePool(pool.getUuid(), info); diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh index 84124d6619b9..f3bfef860962 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh @@ -98,11 +98,11 @@ delete_cephConf() { write_hbLog() { #write the heart beat log timestamp=$(date +%s) - obj=$(rados -p $PoolName ls | grep hb_$HostIP) + obj=$(rados -p $PoolName ls | grep hb-$HostIP) if [ $? -gt 0 ]; then - rados -p $PoolName create hb_$HostIP + rados -p $PoolName create hb-$HostIP fi - echo $timestamp | rados -p $PoolName put hb_$HostIP - + echo $timestamp | rados -p $PoolName put hb-$HostIP - if [ $? -gt 0 ]; then printf "Failed to create rbd file" return 2 @@ -113,9 +113,8 @@ write_hbLog() { check_hbLog() { #check the heart beat log now=$(date +%s) - #$(rados -p $PoolName get hb_$HostIP hb-$HostIP | cat hb-$HostIP) - hb=$(rados -p $PoolName get hb_$HostIP -) - diff=`expr $now - $hb` + hb=$(rados -p $PoolName get hb-$HostIP -) + diff=$(expr $now - $hb) if [ $diff -gt $interval ]; then return $diff fi diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh index 2cc031c76ec7..dc4d6cd2fa49 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh @@ -91,8 +91,7 @@ fi # First check: heartbeat file now=$(date +%s) -#hb=$(rados -p $PoolName get hb_$HostIP hb_$HostIP | cat hb_$HostIP) -hb=$(rados -p $PoolName get hb_$HostIP -) +hb=$(rados -p $PoolName get hb-$HostIP -) diff=$(expr $now - $hb) if [ $diff -lt 61 ]; then echo "=====> ALIVE <=====" @@ -114,21 +113,21 @@ for UUID in $(echo $UUIDList | sed 's/,/ /g'); do done latestUpdateTime=$(echo -e $lastestUUIDList 2> /dev/null | sort -nr | head -1) -obj=$(rados -p $PoolName ls | grep ac_$HostIP) +obj=$(rados -p $PoolName ls | grep ac-$HostIP) if [ $? -gt 0 ]; then - rados -p $PoolName create ac_$HostIP - echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac_$HostIP - + rados -p $PoolName create ac-$HostIP + echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - if [[ $latestUpdateTime -gt $SuspectTime ]]; then echo "=====> ALIVE <=====" else echo "=====> Considering host as DEAD due to file [RBD pool] does not exists and condition [latestUpdateTime -gt SuspectTime] has not been satisfied. <======" fi else - acTime=$(rados -p $PoolName get ac_$HostIP -) + acTime=$(rados -p $PoolName get ac-$HostIP -) arrTime=(${acTime//:/ }) lastSuspectTime=${arrTime[0]} lastUpdateTime=${arrTime[1]} - echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac_$HostIP - + echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - suspectTimeDiff=$(expr $SuspectTime - $lastSuspectTime) if [ $suspectTimeDiff -lt 0 ]; then if [[ $latestUpdateTime -gt $SuspectTime ]]; then From c1ea01028855a8d4c003affe9be9b4c79ac817ce Mon Sep 17 00:00:00 2001 From: sjyu Date: Thu, 9 Dec 2021 11:43:50 +0900 Subject: [PATCH 09/38] =?UTF-8?q?Host=20HA=20:=20RBD=20Primary=20Storage?= =?UTF-8?q?=EC=97=90=20=EB=8C=80=ED=95=9C=20Host=20HA=20=EA=B8=B0=EB=8A=A5?= =?UTF-8?q?=20=EC=A0=9C=EA=B3=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit storagepool > pooltype 제거 및 메서드명변경(getRbdMonIpAddress) --- .../hypervisor/kvm/resource/KVMHABase.java | 26 ++++++++++++++----- .../hypervisor/kvm/resource/KVMHAChecker.java | 19 +------------- .../hypervisor/kvm/resource/KVMHAMonitor.java | 19 +------------- .../kvm/resource/KVMHAVMActivityChecker.java | 19 +------------- .../kvm/storage/KVMStoragePoolManager.java | 4 +-- 5 files changed, 24 insertions(+), 63 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 37f485b8b7c2..36d00e70d67a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -17,6 +17,8 @@ package com.cloud.hypervisor.kvm.resource; import java.io.File; +import java.net.InetAddress; +import java.net.UnknownHostException; import org.apache.log4j.Logger; import org.libvirt.LibvirtException; @@ -24,7 +26,6 @@ import org.libvirt.StoragePoolInfo; import org.libvirt.StoragePoolInfo.StoragePoolState; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.OutputInterpreter.AllLinesParser; import com.cloud.utils.script.Script; @@ -49,15 +50,13 @@ public static class NfsStoragePool { String _poolMountSourcePath; String _mountDestPath; PoolType _type; - StoragePoolType _poolType; - public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, StoragePoolType poolType) { + public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type) { _poolUUID = poolUUID; _poolIp = poolIp; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; _type = type; - _poolType = poolType; } } @@ -67,17 +66,15 @@ public static class RbdStoragePool { String _poolMountSourcePath; String _mountDestPath; PoolType _type; - StoragePoolType _poolType; String _poolAuthSecret; String _poolSourceHost; - public RbdStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, StoragePoolType poolType, String poolAuthSecret, String poolSourceHost) { + public RbdStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthSecret, String poolSourceHost) { _poolUUID = poolUUID; _poolIp = poolIp; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; _type = type; - _poolType = poolType; _poolAuthSecret = poolAuthSecret; _poolSourceHost = poolSourceHost; } @@ -207,6 +204,21 @@ protected String runScriptRetry(String cmdString, OutputInterpreter interpreter) return result; } + public static String getRbdMonIpAddress(String sourceHost) { + try { + String[] hostArr = sourceHost.split(","); + String sourceHostIP = ""; + for (String host : hostArr) { + InetAddress addr = InetAddress.getByName(host); + sourceHostIP += addr.getHostAddress() + ","; + } + return sourceHostIP; + } catch (UnknownHostException e) { + s_logger.debug("Failed to get connection: " + e.getMessage()); + return null; + } + } + public Boolean checkingHeartBeat() { // TODO Auto-generated method stub return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 89a864b650e7..2924a5daaabe 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -24,8 +24,6 @@ import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; -import java.net.InetAddress; -import java.net.UnknownHostException; public class KVMHAChecker extends KVMHABase implements Callable { private static final Logger s_logger = Logger.getLogger(KVMHAChecker.class); @@ -40,21 +38,6 @@ public KVMHAChecker(List nfspools, List rbdpools this.hostIp = host; } - public static String getIpAddress(String sourceHost) { - try { - String[] hostArr = sourceHost.split(","); - String sourceHostIP = ""; - for (String host : hostArr) { - InetAddress addr = InetAddress.getByName(host); - sourceHostIP += addr.getHostAddress() + ","; - } - return sourceHostIP; - } catch (UnknownHostException e) { - s_logger.debug("Failed to get connection: " + e.getMessage()); - return null; - } - } - /* * True means heartbeaing is on going, or we can't get it's status. False * means heartbeating is stopped definitely @@ -92,7 +75,7 @@ public Boolean checkingHeartBeat() { for (RbdStoragePool pool : rbdStoragePools) { Script cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); - cmd.add("-i", getIpAddress(pool._poolSourceHost)); + cmd.add("-i", getRbdMonIpAddress(pool._poolSourceHost)); cmd.add("-p", pool._poolMountSourcePath); cmd.add("-s", pool._poolAuthSecret); cmd.add("-h", hostIp); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index f0c79efb0de2..07a2ffe885a8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -31,8 +31,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.net.InetAddress; -import java.net.UnknownHostException; public class KVMHAMonitor extends KVMHABase implements Runnable { @@ -116,21 +114,6 @@ public RbdStoragePool getRbdStoragePool(String uuid) { } } - public static String getIpAddress(String sourceHost) { - try { - String[] hostArr = sourceHost.split(","); - String sourceHostIP = ""; - for (String host : hostArr) { - InetAddress addr = InetAddress.getByName(host); - sourceHostIP += addr.getHostAddress() + ","; - } - return sourceHostIP; - } catch (UnknownHostException e) { - s_logger.debug("Failed to get connection: " + e.getMessage()); - return null; - } - } - protected void runHeartBeat() { if(nfsstoragePool != null && !nfsstoragePool.isEmpty()) { synchronized (nfsstoragePool) { @@ -286,7 +269,7 @@ private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String private Script createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { Script cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); - cmd.add("-i", getIpAddress(primaryStoragePool._poolSourceHost)); + cmd.add("-i", getRbdMonIpAddress(primaryStoragePool._poolSourceHost)); cmd.add("-p", primaryStoragePool._poolMountSourcePath); cmd.add("-s", primaryStoragePool._poolAuthSecret); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 207576a771f9..369358f76108 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -23,8 +23,6 @@ import org.joda.time.Duration; import java.util.concurrent.Callable; -import java.net.InetAddress; -import java.net.UnknownHostException; public class KVMHAVMActivityChecker extends KVMHABase implements Callable { private static final Logger LOG = Logger.getLogger(KVMHAVMActivityChecker.class); @@ -48,21 +46,6 @@ public KVMHAVMActivityChecker(final NfsStoragePool pool, final RbdStoragePool rb this.poolType = poolType; } - public static String getIpAddress(String sourceHost) { - try { - String[] hostArr = sourceHost.split(","); - String sourceHostIP = ""; - for (String host : hostArr) { - InetAddress addr = InetAddress.getByName(host); - sourceHostIP += addr.getHostAddress() + ","; - } - return sourceHostIP; - } catch (UnknownHostException e) { - LOG.debug("Failed to get connection: " + e.getMessage()); - return null; - } - } - @Override public Boolean checkingHeartBeat() { Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), LOG); @@ -73,7 +56,7 @@ public Boolean checkingHeartBeat() { cmd.add("-m", nfsStoragePool._mountDestPath); poolIp = nfsStoragePool._poolIp; } else if (poolType == StoragePoolType.RBD) { - cmd.add("-i", getIpAddress(rbdStoragePool._poolSourceHost)); + cmd.add("-i", getRbdMonIpAddress(rbdStoragePool._poolSourceHost)); cmd.add("-p", rbdStoragePool._poolMountSourcePath); cmd.add("-s", rbdStoragePool._poolAuthSecret); poolIp = rbdStoragePool._poolIp; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 1979ff750dce..e9033f893963 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -331,10 +331,10 @@ private synchronized KVMStoragePool createStoragePool(String name, String host, // LibvirtStorageAdaptor-specific statement if (type == StoragePoolType.NetworkFilesystem && primaryStorage) { - KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.NetworkFilesystem); + KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage); _haMonitor.addStoragePool(nfspool); } else if (type == StoragePoolType.RBD && primaryStorage) { - KVMHABase.RbdStoragePool rbdpool = new KVMHABase.RbdStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, StoragePoolType.RBD, pool.getAuthSecret(), pool.getSourceHost()); + KVMHABase.RbdStoragePool rbdpool = new KVMHABase.RbdStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, pool.getAuthSecret(), pool.getSourceHost()); _haMonitor.addStoragePool(rbdpool); } StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, primaryStorage); From 481afe75c476eee6ddfc9016f6d2895551e41abe Mon Sep 17 00:00:00 2001 From: sjyu Date: Mon, 13 Dec 2021 10:34:15 +0900 Subject: [PATCH 10/38] =?UTF-8?q?Host=20HA=20:=20RBD=20Primary=20Storage?= =?UTF-8?q?=EC=97=90=20=EB=8C=80=ED=95=9C=20Host=20HA=20=EA=B8=B0=EB=8A=A5?= =?UTF-8?q?=20=EC=A0=9C=EA=B3=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rbd pool auth 변경가능하도록 수정 --- .../hypervisor/kvm/resource/KVMHABase.java | 13 ++++++++--- .../hypervisor/kvm/resource/KVMHAChecker.java | 1 + .../hypervisor/kvm/resource/KVMHAMonitor.java | 1 + .../kvm/resource/KVMHAVMActivityChecker.java | 1 + .../kvm/storage/KVMStoragePoolManager.java | 2 +- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh | 19 +++++++++------ .../vm/hypervisor/kvm/kvmvmactivity_rbd.sh | 23 +++++++++++-------- 7 files changed, 40 insertions(+), 20 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 36d00e70d67a..7ed9dc186449 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -66,15 +66,17 @@ public static class RbdStoragePool { String _poolMountSourcePath; String _mountDestPath; PoolType _type; + String _poolAuthUserName; String _poolAuthSecret; String _poolSourceHost; - public RbdStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthSecret, String poolSourceHost) { + public RbdStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthUserName, String poolAuthSecret, String poolSourceHost) { _poolUUID = poolUUID; _poolIp = poolIp; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; _type = type; + _poolAuthUserName = poolAuthUserName; _poolAuthSecret = poolAuthSecret; _poolSourceHost = poolSourceHost; } @@ -209,8 +211,13 @@ public static String getRbdMonIpAddress(String sourceHost) { String[] hostArr = sourceHost.split(","); String sourceHostIP = ""; for (String host : hostArr) { - InetAddress addr = InetAddress.getByName(host); - sourceHostIP += addr.getHostAddress() + ","; + String ipRegex = "(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])"; + if (host.matches(ipRegex)) { + sourceHostIP += host + ","; + } else { + InetAddress addr = InetAddress.getByName(host); + sourceHostIP += addr.getHostAddress() + ","; + } } return sourceHostIP; } catch (UnknownHostException e) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 2924a5daaabe..56f8368849cd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -77,6 +77,7 @@ public Boolean checkingHeartBeat() { Script cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); cmd.add("-i", getRbdMonIpAddress(pool._poolSourceHost)); cmd.add("-p", pool._poolMountSourcePath); + cmd.add("-n", pool._poolAuthUserName); cmd.add("-s", pool._poolAuthSecret); cmd.add("-h", hostIp); cmd.add("-r"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 07a2ffe885a8..c791fc758891 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -271,6 +271,7 @@ private Script createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, Stri Script cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); cmd.add("-i", getRbdMonIpAddress(primaryStoragePool._poolSourceHost)); cmd.add("-p", primaryStoragePool._poolMountSourcePath); + cmd.add("-n", primaryStoragePool._poolAuthUserName); cmd.add("-s", primaryStoragePool._poolAuthSecret); if (hostValidation) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 369358f76108..094424cdcb68 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -58,6 +58,7 @@ public Boolean checkingHeartBeat() { } else if (poolType == StoragePoolType.RBD) { cmd.add("-i", getRbdMonIpAddress(rbdStoragePool._poolSourceHost)); cmd.add("-p", rbdStoragePool._poolMountSourcePath); + cmd.add("-n", rbdStoragePool._poolAuthUserName); cmd.add("-s", rbdStoragePool._poolAuthSecret); poolIp = rbdStoragePool._poolIp; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index e9033f893963..7bb9faffaa5f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -334,7 +334,7 @@ private synchronized KVMStoragePool createStoragePool(String name, String host, KVMHABase.NfsStoragePool nfspool = new KVMHABase.NfsStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage); _haMonitor.addStoragePool(nfspool); } else if (type == StoragePoolType.RBD && primaryStorage) { - KVMHABase.RbdStoragePool rbdpool = new KVMHABase.RbdStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, pool.getAuthSecret(), pool.getSourceHost()); + KVMHABase.RbdStoragePool rbdpool = new KVMHABase.RbdStoragePool(pool.getUuid(), host, path, pool.getLocalPath(), PoolType.PrimaryStorage, pool.getAuthUserName(), pool.getAuthSecret(), pool.getSourceHost()); _haMonitor.addStoragePool(rbdpool); } StoragePoolInformation info = new StoragePoolInformation(name, host, port, path, userInfo, type, primaryStorage); diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh index f3bfef860962..3a4052dcbe8c 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh @@ -19,6 +19,7 @@ help() { printf "Usage: $0 -p rbd pool name + -n pool auth username -s pool auth secret -h host -i source host ip @@ -29,6 +30,7 @@ help() { } #set -x PoolName= +PoolAuthUserName= PoolAuthSecret= HostIP= SourceHostIP= @@ -36,12 +38,15 @@ interval= rflag=0 cflag=0 -while getopts 'p:s:h:i:t:rc' OPTION +while getopts 'p:n:s:h:i:t:rc' OPTION do case $OPTION in p) PoolName="$OPTARG" ;; + n) + PoolAuthUserName="$OPTARG" + ;; s) PoolAuthSecret="$OPTARG" ;; @@ -70,13 +75,13 @@ if [ -z "$PoolName" ]; then exit 2 fi -keyringFile="/etc/ceph/keyring.bin" +keyringFile="/etc/ceph/keyring" confFile="/etc/ceph/ceph.conf" create_cephConf() { #Creating Ceph keyring and conf for executing rados commands if [ ! -f $keyringFile ]; then - echo -e "[client.admin]\n key=$PoolAuthSecret" > $keyringFile + echo -e "[client.$PoolAuthUserName]\n key=$PoolAuthSecret" > $keyringFile fi if [ ! -f $confFile ]; then @@ -98,11 +103,11 @@ delete_cephConf() { write_hbLog() { #write the heart beat log timestamp=$(date +%s) - obj=$(rados -p $PoolName ls | grep hb-$HostIP) + obj=$(rados -p $PoolName ls --id $PoolAuthUserName | grep hb-$HostIP) if [ $? -gt 0 ]; then - rados -p $PoolName create hb-$HostIP + rados -p $PoolName create hb-$HostIP --id $PoolAuthUserName fi - echo $timestamp | rados -p $PoolName put hb-$HostIP - + echo $timestamp | rados -p $PoolName put hb-$HostIP - --id $PoolAuthUserName if [ $? -gt 0 ]; then printf "Failed to create rbd file" return 2 @@ -113,7 +118,7 @@ write_hbLog() { check_hbLog() { #check the heart beat log now=$(date +%s) - hb=$(rados -p $PoolName get hb-$HostIP -) + hb=$(rados -p $PoolName get hb-$HostIP - --id $PoolAuthUserName) diff=$(expr $now - $hb) if [ $diff -gt $interval ]; then return $diff diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh index dc4d6cd2fa49..db64ae39f71d 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh @@ -18,6 +18,7 @@ help() { printf "Usage: $0 -p rbd pool name + -n pool auth username -s pool auth secret -h host -i source host ip @@ -28,6 +29,7 @@ help() { } #set -x PoolName= +PoolAuthUserName= PoolAuthSecret= HostIP= SourceHostIP= @@ -35,12 +37,15 @@ UUIDList= MSTime= SuspectTime= -while getopts 'p:s:h:i:u:t:d:' OPTION +while getopts 'p:n:s:h:i:u:t:d:' OPTION do case $OPTION in p) PoolName="$OPTARG" ;; + n) + PoolAuthUserName="$OPTARG" + ;; s) PoolAuthSecret="$OPTARG" ;; @@ -78,7 +83,7 @@ keyringFile="/etc/ceph/keyring.bin" confFile="/etc/ceph/ceph.conf" if [ ! -f $keyringFile ]; then - echo -e "[client.admin]\n key=$PoolAuthSecret" > $keyringFile + echo -e "[client.$PoolAuthUserName]\n key=$PoolAuthSecret" > $keyringFile fi if [ ! -f $confFile ]; then @@ -91,7 +96,7 @@ fi # First check: heartbeat file now=$(date +%s) -hb=$(rados -p $PoolName get hb-$HostIP -) +hb=$(rados -p $PoolName get hb-$HostIP - --id $PoolAuthUserName) diff=$(expr $now - $hb) if [ $diff -lt 61 ]; then echo "=====> ALIVE <=====" @@ -106,28 +111,28 @@ fi # Second check: disk activity check lastestUUIDList= for UUID in $(echo $UUIDList | sed 's/,/ /g'); do - time=$(rbd info $UUID | grep modify_timestamp) + time=$(rbd info $UUID --id $PoolAuthUserName | grep modify_timestamp) time=${time#*modify_timestamp: } time=$(date -d "$time" +%s) lastestUUIDList+="${time}\n" done latestUpdateTime=$(echo -e $lastestUUIDList 2> /dev/null | sort -nr | head -1) -obj=$(rados -p $PoolName ls | grep ac-$HostIP) +obj=$(rados -p $PoolName ls --id $PoolAuthUserName | grep ac-$HostIP) if [ $? -gt 0 ]; then - rados -p $PoolName create ac-$HostIP - echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - + rados -p $PoolName create ac-$HostIP --id $PoolAuthUserName + echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - --id $PoolAuthUserName if [[ $latestUpdateTime -gt $SuspectTime ]]; then echo "=====> ALIVE <=====" else echo "=====> Considering host as DEAD due to file [RBD pool] does not exists and condition [latestUpdateTime -gt SuspectTime] has not been satisfied. <======" fi else - acTime=$(rados -p $PoolName get ac-$HostIP -) + acTime=$(rados -p $PoolName get ac-$HostIP - --id $PoolAuthUserName) arrTime=(${acTime//:/ }) lastSuspectTime=${arrTime[0]} lastUpdateTime=${arrTime[1]} - echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - + echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - --id $PoolAuthUserName suspectTimeDiff=$(expr $SuspectTime - $lastSuspectTime) if [ $suspectTimeDiff -lt 0 ]; then if [[ $latestUpdateTime -gt $SuspectTime ]]; then From 4dcd1416e1f1d0306c32795041e301ade9301fb1 Mon Sep 17 00:00:00 2001 From: sjyu Date: Wed, 15 Dec 2021 14:54:00 +0900 Subject: [PATCH 11/38] =?UTF-8?q?Host=20HA=20:=20RBD=20Primary=20Storage?= =?UTF-8?q?=EC=97=90=20=EB=8C=80=ED=95=9C=20Host=20HA=20=EA=B8=B0=EB=8A=A5?= =?UTF-8?q?=20=EC=A0=9C=EA=B3=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nfs, rbd Storage hb모니터링 함께 되도록 수정 --- .../java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index c791fc758891..24bc058c41ec 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -181,7 +181,9 @@ protected void runHeartBeat() { } } } - } else if (rbdstoragePool != null && !rbdstoragePool.isEmpty()) { + } + + if (rbdstoragePool != null && !rbdstoragePool.isEmpty()) { synchronized (rbdstoragePool) { Set removedPools = new HashSet<>(); for (String uuid : rbdstoragePool.keySet()) { From 9c80a17a58dbd2c70df8c3c8544214e54d38202d Mon Sep 17 00:00:00 2001 From: sjyu Date: Fri, 28 Jan 2022 14:28:03 +0900 Subject: [PATCH 12/38] =?UTF-8?q?HA=20Cloudstack=20=EC=9A=94=EC=B2=AD?= =?UTF-8?q?=EC=82=AC=ED=95=AD=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 코딩규칙 및 요청사항 수정 --- .../hypervisor/kvm/resource/KVMHABase.java | 4 +- .../hypervisor/kvm/resource/KVMHAMonitor.java | 176 +++++++----------- ...VMActivityOnStoragePoolCommandWrapper.java | 7 +- .../wrapper/LibvirtFenceCommandWrapper.java | 6 +- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 4 +- 5 files changed, 83 insertions(+), 114 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 7ed9dc186449..d7296110dab2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -37,7 +37,7 @@ public class KVMHABase { protected static String s_heartBeatPathRbd; protected long _heartBeatUpdateTimeout = 60000; protected long _heartBeatUpdateFreq = 60000; - protected long _heartBeatUpdateMaxTries = 5; + protected long _heartBeatUpdateMaxTries = 10; protected long _heartBeatUpdateRetrySleep = 10000; public static enum PoolType { @@ -221,7 +221,7 @@ public static String getRbdMonIpAddress(String sourceHost) { } return sourceHostIP; } catch (UnknownHostException e) { - s_logger.debug("Failed to get connection: " + e.getMessage()); + s_logger.warn("Failed to get connection: " + e); return null; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 24bc058c41ec..bc8e783038e5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -115,64 +115,13 @@ public RbdStoragePool getRbdStoragePool(String uuid) { } protected void runHeartBeat() { - if(nfsstoragePool != null && !nfsstoragePool.isEmpty()) { + if (nfsstoragePool != null && !nfsstoragePool.isEmpty()) { synchronized (nfsstoragePool) { Set removedPools = new HashSet<>(); for (String uuid : nfsstoragePool.keySet()) { NfsStoragePool primaryStoragePool = nfsstoragePool.get(uuid); - StoragePool storage; - try { - Connect conn = LibvirtConnection.getConnection(); - storage = conn.storagePoolLookupByUUIDString(uuid); - if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - if (storage == null) { - s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); - } else { - s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); - } - - removedPools.add(uuid); - continue; - } - - s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); - - } catch (LibvirtException e) { - s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); - - if (e.toString().contains("pool not found")) { - s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); - removedPools.add(uuid); - continue; - } - - } - - String result = null; - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { - Script cmd = createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - result = cmd.execute(); - - s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); - - if (result != null) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { - break; - } - - } - - if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); - Script cmd = createHeartBeatCommand(primaryStoragePool, null, false); - result = cmd.execute(); - } + runHeartBeatCommon(primaryStoragePool, null, uuid, removedPools); + continue; } if (!removedPools.isEmpty()) { @@ -188,68 +137,85 @@ protected void runHeartBeat() { Set removedPools = new HashSet<>(); for (String uuid : rbdstoragePool.keySet()) { RbdStoragePool primaryStoragePool = rbdstoragePool.get(uuid); - StoragePool storage; - try { - Connect conn = LibvirtConnection.getConnection(); - storage = conn.storagePoolLookupByUUIDString(uuid); - if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - if (storage == null) { - s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); - } else { - s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); - } - - removedPools.add(uuid); - continue; - } - - s_logger.debug(String.format("Found RBD storage pool [%s] in libvirt, continuing.", uuid)); - - } catch (LibvirtException e) { - s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); - - if (e.toString().contains("pool not found")) { - s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); - removedPools.add(uuid); - continue; - } + runHeartBeatCommon(null, primaryStoragePool, uuid, removedPools); + continue; + } + if (!removedPools.isEmpty()) { + for (String uuid : removedPools) { + removeRbdStoragePool(uuid); } + } + } + } + } + + private Set runHeartBeatCommon(NfsStoragePool nfsStoragePool, RbdStoragePool rbdStoragePool, String uuid, Set removedPools) { + StoragePool storage; + try { + Connect conn = LibvirtConnection.getConnection(); + storage = conn.storagePoolLookupByUUIDString(uuid); + if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { + if (storage == null) { + s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + } else { + s_logger.debug(String.format("Libvirt storage pool [%s] was found, but it is not running, removing it from HA list.", uuid)); + } - String result = null; - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { - Script cmd = createRbdHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - result = cmd.execute(); + removedPools.add(uuid); + } - s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); + s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); - if (result != null) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { - break; - } + } catch (LibvirtException e) { + s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); - } + if (e.toString().contains("pool not found")) { + s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + removedPools.add(uuid); + } - if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); - Script cmd = createRbdHeartBeatCommand(primaryStoragePool, null, false); - result = cmd.execute(); - } - } + } - if (!removedPools.isEmpty()) { - for (String uuid : removedPools) { - removeRbdStoragePool(uuid); - } + String result = null; + for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + Script cmd; + if (nfsStoragePool != null) { + cmd = createHeartBeatCommand(nfsStoragePool, hostPrivateIp, true); + result = cmd.execute(); + s_logger.debug(String.format("The command [%s], to the pool [%s], had the result [%s].", cmd.toString(), uuid, result)); + } else if (rbdStoragePool != null) { + cmd = createRbdHeartBeatCommand(rbdStoragePool, hostPrivateIp, true); + result = cmd.execute(); + s_logger.debug(String.format("The command [%s], to the pool [%s], had the result [%s].", cmd.toString(), uuid, result)); + } + + if (result != null) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); + try { + Thread.sleep(_heartBeatUpdateRetrySleep); + } catch (InterruptedException e) { + s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); } + } else { + break; } + } + + if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + Script cmd; + if (nfsStoragePool != null) { + cmd = createHeartBeatCommand(nfsStoragePool, null, false); + result = cmd.execute(); + } else if (rbdStoragePool != null) { + cmd = createRbdHeartBeatCommand(rbdStoragePool, null, false); + result = cmd.execute(); + } + } + + return removedPools; } private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index da4c90174dab..f0a89d83524c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -48,8 +48,11 @@ public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final L final NfsStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); final RbdStoragePool rbdpool = monitor.getRbdStoragePool(pool.getUuid()); String vmActivityCheckPath = ""; - if (Storage.StoragePoolType.NetworkFilesystem == pool.getType()) vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPath(); - else if (Storage.StoragePoolType.RBD == pool.getType()) vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPathRbd(); + if (Storage.StoragePoolType.NetworkFilesystem == pool.getType()) { + vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPath(); + } else if (Storage.StoragePoolType.RBD == pool.getType()) { + vmActivityCheckPath = libvirtComputingResource.getVmActivityCheckPathRbd(); + } final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, rbdpool, command.getHost().getPrivateNetwork().getIp(), command.getVolumeList(), vmActivityCheckPath, command.getSuspectTimeInSeconds(), pool.getType()); final Future future = executors.submit(ha); try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java index 4916c8c7c1fe..9a676220e33d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtFenceCommandWrapper.java @@ -57,11 +57,11 @@ public Answer execute(final FenceCommand command, final LibvirtComputingResource * a heartbeat. Disable Fencing Off for hosts without NFS */ if (nfspools.size() == 0) { - String logline = "No NFS storage pools found. No way to safely fence " + command.getVmName() + " on host " + command.getHostGuid(); + String logline = String.format("No NFS storage pools found. No way to safely fence %s on host %s", command.getVmName(), command.getHostGuid()); s_logger.warn(logline); return new FenceAnswer(command, false, logline); - }else if (rbdpools.size() == 0) { - String logline = "No RBD storage pools found. No way to safely fence " + command.getVmName() + " on host " + command.getHostGuid(); + } else if (rbdpools.size() == 0) { + String logline = String.format("No RBD storage pools found. No way to safely fence %s on host %s", command.getVmName(), command.getHostGuid()); s_logger.warn(logline); return new FenceAnswer(command, false, logline); } diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index a08e4c519ff0..46fb457c2f5d 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -177,7 +177,7 @@ public DataStore initialize(Map dsInfos) { String multiHost = uriHost + (uriPath.substring(0, uriPath.lastIndexOf("/")).replaceAll("/", ",")); String[] hostArr = multiHost.split(","); if (hostArr.length > 5) { - throw new InvalidParameterValueException("RADOS monitor can support up to 5."); + throw new InvalidParameterValueException("RADOS monitor can support up to 5 hosts."); } } } else if (uri.getScheme().equalsIgnoreCase("gluster")) { @@ -259,7 +259,7 @@ public DataStore initialize(Map dsInfos) { } if (multi) { storageHost = storageHost + (hostPath.substring(0, hostPath.lastIndexOf("/")).replaceAll("/", ",")); - hostPath = hostPath.substring(hostPath.lastIndexOf("/")+1); + hostPath = hostPath.substring(hostPath.lastIndexOf("/") + 1); } parameters.setType(StoragePoolType.RBD); parameters.setHost(storageHost); From 8a71e4845b6bda48c6afe719ad070cab9cb35c46 Mon Sep 17 00:00:00 2001 From: sjyu Date: Fri, 28 Jan 2022 15:03:50 +0900 Subject: [PATCH 13/38] =?UTF-8?q?HA=20Cloudstack=20=EC=9A=94=EC=B2=AD?= =?UTF-8?q?=EC=82=AC=ED=95=AD=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runHeartbeatToPool 메서드명 수정 --- .../cloud/hypervisor/kvm/resource/KVMHAMonitor.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index bc8e783038e5..f2f6199958de 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -37,6 +37,8 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private static final Logger s_logger = Logger.getLogger(KVMHAMonitor.class); private final Map nfsstoragePool = new ConcurrentHashMap<>(); private final Map rbdstoragePool = new ConcurrentHashMap<>(); + private final NfsStoragePool nfsStoragePool = null; + private final RbdStoragePool rbdStoragePool = null; private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; @@ -119,8 +121,8 @@ protected void runHeartBeat() { synchronized (nfsstoragePool) { Set removedPools = new HashSet<>(); for (String uuid : nfsstoragePool.keySet()) { - NfsStoragePool primaryStoragePool = nfsstoragePool.get(uuid); - runHeartBeatCommon(primaryStoragePool, null, uuid, removedPools); + NfsStoragePool nfsStoragePool = nfsstoragePool.get(uuid); + runHeartbeatToPool(nfsStoragePool, rbdStoragePool, uuid, removedPools); continue; } @@ -136,8 +138,8 @@ protected void runHeartBeat() { synchronized (rbdstoragePool) { Set removedPools = new HashSet<>(); for (String uuid : rbdstoragePool.keySet()) { - RbdStoragePool primaryStoragePool = rbdstoragePool.get(uuid); - runHeartBeatCommon(null, primaryStoragePool, uuid, removedPools); + RbdStoragePool rbdStoragePool = rbdstoragePool.get(uuid); + runHeartbeatToPool(nfsStoragePool, rbdStoragePool, uuid, removedPools); continue; } @@ -150,7 +152,7 @@ protected void runHeartBeat() { } } - private Set runHeartBeatCommon(NfsStoragePool nfsStoragePool, RbdStoragePool rbdStoragePool, String uuid, Set removedPools) { + private Set runHeartbeatToPool(NfsStoragePool nfsStoragePool, RbdStoragePool rbdStoragePool, String uuid, Set removedPools) { StoragePool storage; try { Connect conn = LibvirtConnection.getConnection(); From 29a544c8b91e60f216984a0daabcf40142805d23 Mon Sep 17 00:00:00 2001 From: sjyu Date: Fri, 18 Feb 2022 12:49:43 +0900 Subject: [PATCH 14/38] =?UTF-8?q?rbd=20watcher=20=EB=B0=A9=EC=8B=9D?= =?UTF-8?q?=EC=9C=BC=EB=A1=9C=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 각 호스트용 rbd 이미지(hb-hostip) 생성하여 watcher 감시 2. /etc/ceph 파일생성 대신 -c, -k 옵션을 사용하여 rbd 실행 3. keyring파일은 /etc/cloudstack/agent/ 경로에 생성 --- .../hypervisor/kvm/resource/KVMHABase.java | 22 ----- .../hypervisor/kvm/resource/KVMHAChecker.java | 3 +- .../hypervisor/kvm/resource/KVMHAMonitor.java | 2 +- .../kvm/resource/KVMHAVMActivityChecker.java | 6 +- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh | 83 ++++++++----------- .../vm/hypervisor/kvm/kvmvmactivity_rbd.sh | 82 ++++-------------- 6 files changed, 55 insertions(+), 143 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index d7296110dab2..e26262b9f530 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -17,8 +17,6 @@ package com.cloud.hypervisor.kvm.resource; import java.io.File; -import java.net.InetAddress; -import java.net.UnknownHostException; import org.apache.log4j.Logger; import org.libvirt.LibvirtException; @@ -206,26 +204,6 @@ protected String runScriptRetry(String cmdString, OutputInterpreter interpreter) return result; } - public static String getRbdMonIpAddress(String sourceHost) { - try { - String[] hostArr = sourceHost.split(","); - String sourceHostIP = ""; - for (String host : hostArr) { - String ipRegex = "(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])\\.(\\d{1,2}|1\\d\\d|2[0-4]\\d|25[0-5])"; - if (host.matches(ipRegex)) { - sourceHostIP += host + ","; - } else { - InetAddress addr = InetAddress.getByName(host); - sourceHostIP += addr.getHostAddress() + ","; - } - } - return sourceHostIP; - } catch (UnknownHostException e) { - s_logger.warn("Failed to get connection: " + e); - return null; - } - } - public Boolean checkingHeartBeat() { // TODO Auto-generated method stub return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 56f8368849cd..292eb3c02ea0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -75,13 +75,12 @@ public Boolean checkingHeartBeat() { for (RbdStoragePool pool : rbdStoragePools) { Script cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); - cmd.add("-i", getRbdMonIpAddress(pool._poolSourceHost)); + cmd.add("-i", pool._poolSourceHost); cmd.add("-p", pool._poolMountSourcePath); cmd.add("-n", pool._poolAuthUserName); cmd.add("-s", pool._poolAuthSecret); cmd.add("-h", hostIp); cmd.add("-r"); - cmd.add("-t", String.valueOf(_heartBeatUpdateFreq / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index f2f6199958de..af60ea7b129c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -239,7 +239,7 @@ private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String private Script createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { Script cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); - cmd.add("-i", getRbdMonIpAddress(primaryStoragePool._poolSourceHost)); + cmd.add("-i", primaryStoragePool._poolSourceHost); cmd.add("-p", primaryStoragePool._poolMountSourcePath); cmd.add("-n", primaryStoragePool._poolAuthUserName); cmd.add("-s", primaryStoragePool._poolAuthSecret); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 094424cdcb68..8c6501acc9ab 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -54,9 +54,11 @@ public Boolean checkingHeartBeat() { cmd.add("-i", nfsStoragePool._poolIp); cmd.add("-p", nfsStoragePool._poolMountSourcePath); cmd.add("-m", nfsStoragePool._mountDestPath); + cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000))); + cmd.add("-d", String.valueOf(suspectTimeInSeconds)); poolIp = nfsStoragePool._poolIp; } else if (poolType == StoragePoolType.RBD) { - cmd.add("-i", getRbdMonIpAddress(rbdStoragePool._poolSourceHost)); + cmd.add("-i", rbdStoragePool._poolSourceHost); cmd.add("-p", rbdStoragePool._poolMountSourcePath); cmd.add("-n", rbdStoragePool._poolAuthUserName); cmd.add("-s", rbdStoragePool._poolAuthSecret); @@ -64,8 +66,6 @@ public Boolean checkingHeartBeat() { } cmd.add("-h", hostIP); cmd.add("-u", volumeUuidList); - cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000))); - cmd.add("-d", String.valueOf(suspectTimeInSeconds)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh index 3a4052dcbe8c..b4e1b9cc3c44 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh @@ -23,9 +23,8 @@ help() { -s pool auth secret -h host -i source host ip - -r write/read hb log - -c cleanup - -t interval between read hb log\n" + -r cretae/read hb watcher + -c cleanup" exit 1 } #set -x @@ -34,11 +33,10 @@ PoolAuthUserName= PoolAuthSecret= HostIP= SourceHostIP= -interval= rflag=0 cflag=0 -while getopts 'p:n:s:h:i:t:rc' OPTION +while getopts 'p:n:s:h:i:rc' OPTION do case $OPTION in p) @@ -56,9 +54,6 @@ do i) SourceHostIP="$OPTARG" ;; - t) - interval="$OPTARG" - ;; r) rflag=1 ;; @@ -75,77 +70,67 @@ if [ -z "$PoolName" ]; then exit 2 fi -keyringFile="/etc/ceph/keyring" -confFile="/etc/ceph/ceph.conf" +keyringFile="/etc/cloudstack/agent/keyring" -create_cephConf() { -#Creating Ceph keyring and conf for executing rados commands +create_cephKeyring() { +#Creating Ceph keyring for executing rbd commands if [ ! -f $keyringFile ]; then echo -e "[client.$PoolAuthUserName]\n key=$PoolAuthSecret" > $keyringFile fi - - if [ ! -f $confFile ]; then - confContents="[global]\n mon host = " - for ip in $(echo $SourceHostIP | sed 's/,/ /g'); do - confContents+="[v2:${ip}:3300/0,v1:${ip}:6789/0], " - done - echo -e "$confContents" | sed 's/, $//' > $confFile - fi } -delete_cephConf() { +delete_cephKeyring() { #Deleting Ceph keyring if [ -f $keyringFile ]; then rm -rf $keyringFile fi } -write_hbLog() { -#write the heart beat log - timestamp=$(date +%s) - obj=$(rados -p $PoolName ls --id $PoolAuthUserName | grep hb-$HostIP) - if [ $? -gt 0 ]; then - rados -p $PoolName create hb-$HostIP --id $PoolAuthUserName +cretae_hbWatcher() { +#Create HB RBD Image and watcher + status=$(rbd status hb-$HostIP -m $SourceHostIP -k $keyringFile) + if [ $? == 2 ]; then + rbd create hb-$HostIP --size 1 -m $SourceHostIP -k $keyringFile + setsid sh -c 'exec rbd watch hb-'$HostIP' -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' fi - echo $timestamp | rados -p $PoolName put hb-$HostIP - --id $PoolAuthUserName - if [ $? -gt 0 ]; then - printf "Failed to create rbd file" - return 2 + + if [ "$status" == "Watchers: none" ]; then + setsid sh -c 'exec rbd watch hb-'$HostIP' -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' fi + return 0 } -check_hbLog() { -#check the heart beat log - now=$(date +%s) - hb=$(rados -p $PoolName get hb-$HostIP - --id $PoolAuthUserName) - diff=$(expr $now - $hb) - if [ $diff -gt $interval ]; then - return $diff +check_hbWatcher() { +#check the heart beat watcher + hb=$(rbd status hb-$HostIP -m $SourceHostIP -k $keyringFile) + if [ "$hb" == "Watchers: none" ]; then + return 2 + else + return 0 fi - return 0 } if [ "$rflag" == "1" ]; then - create_cephConf - check_hbLog - diff=$? - if [ $diff == 0 ]; then + create_cephKeyring + check_hbWatcher + hb=$? + if [ "$hb" != "Watchers: none" ]; then echo "=====> ALIVE <=====" else - echo "=====> Considering host as DEAD because last write on [RBD pool] was [$diff] seconds ago, but the max interval is [$interval] <======" + echo "=====> Considering host as DEAD due to [hb-$HostIP] watcher that the host is seeing is not running. <======" fi - delete_cephConf + delete_cephKeyring exit 0 elif [ "$cflag" == "1" ]; then - /usr/bin/logger -t heartbeat "kvmheartbeat_rbd.sh will reboot system because it was unable to write the heartbeat to the storage." + /usr/bin/logger -t heartbeat "kvmheartbeat_rbd.sh reboots the system because there is no heartbeat watcher." sync & sleep 5 echo b > /proc/sysrq-trigger exit $? else - create_cephConf - write_hbLog - delete_cephConf + create_cephKeyring + cretae_hbWatcher + delete_cephKeyring exit 0 fi diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh index db64ae39f71d..795e36d1359b 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh @@ -22,9 +22,7 @@ help() { -s pool auth secret -h host -i source host ip - -u volume uuid list - -t time on ms - -d suspect time\n" + -u volume uuid list" exit 1 } #set -x @@ -34,10 +32,8 @@ PoolAuthSecret= HostIP= SourceHostIP= UUIDList= -MSTime= -SuspectTime= -while getopts 'p:n:s:h:i:u:t:d:' OPTION +while getopts 'p:n:s:h:i:u:d:' OPTION do case $OPTION in p) @@ -58,12 +54,6 @@ do u) UUIDList="$OPTARG" ;; - t) - MSTime="$OPTARG" - ;; - d) - SuspectTime="$OPTARG" - ;; *) help ;; @@ -74,31 +64,16 @@ if [ -z "$PoolName" ]; then exit 2 fi -if [ -z "$SuspectTime" ]; then - exit 2 -fi - -#Creating Ceph keyring and conf for executing rados commands -keyringFile="/etc/ceph/keyring.bin" -confFile="/etc/ceph/ceph.conf" +#Creating Ceph keyring for executing rbd commands +keyringFile="/etc/cloudstack/agent/keyring.bin" if [ ! -f $keyringFile ]; then echo -e "[client.$PoolAuthUserName]\n key=$PoolAuthSecret" > $keyringFile fi -if [ ! -f $confFile ]; then - confContents="[global]\n mon host = " - for ip in $(echo $SourceHostIP | sed 's/,/ /g'); do - confContents+="[v2:${ip}:3300/0,v1:${ip}:6789/0], " - done - echo -e "$confContents" | sed 's/, $//' > $confFile -fi - -# First check: heartbeat file -now=$(date +%s) -hb=$(rados -p $PoolName get hb-$HostIP - --id $PoolAuthUserName) -diff=$(expr $now - $hb) -if [ $diff -lt 61 ]; then +# First check: heartbeat watcher +status=$(rbd status hb-$HostIP -m $SourceHostIP -k $keyringFile) +if [ "$status" != "Watchers: none" ]; then echo "=====> ALIVE <=====" exit 0 fi @@ -109,44 +84,19 @@ if [ -z "$UUIDList" ]; then fi # Second check: disk activity check -lastestUUIDList= +statusFlag=true for UUID in $(echo $UUIDList | sed 's/,/ /g'); do - time=$(rbd info $UUID --id $PoolAuthUserName | grep modify_timestamp) - time=${time#*modify_timestamp: } - time=$(date -d "$time" +%s) - lastestUUIDList+="${time}\n" + diskStatus=$(rbd status $UUID -m $SourceHostIP -k $keyringFile) + if [ "$status" == "Watchers: none" ]; then + statusFlag=false + break + fi done -latestUpdateTime=$(echo -e $lastestUUIDList 2> /dev/null | sort -nr | head -1) -obj=$(rados -p $PoolName ls --id $PoolAuthUserName | grep ac-$HostIP) -if [ $? -gt 0 ]; then - rados -p $PoolName create ac-$HostIP --id $PoolAuthUserName - echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - --id $PoolAuthUserName - if [[ $latestUpdateTime -gt $SuspectTime ]]; then - echo "=====> ALIVE <=====" - else - echo "=====> Considering host as DEAD due to file [RBD pool] does not exists and condition [latestUpdateTime -gt SuspectTime] has not been satisfied. <======" - fi +if [ statusFlag == "true" ]; then + echo "=====> ALIVE <=====" else - acTime=$(rados -p $PoolName get ac-$HostIP - --id $PoolAuthUserName) - arrTime=(${acTime//:/ }) - lastSuspectTime=${arrTime[0]} - lastUpdateTime=${arrTime[1]} - echo "$SuspectTime:$latestUpdateTime:$MSTime" | rados -p $PoolName put ac-$HostIP - --id $PoolAuthUserName - suspectTimeDiff=$(expr $SuspectTime - $lastSuspectTime) - if [ $suspectTimeDiff -lt 0 ]; then - if [[ $latestUpdateTime -gt $SuspectTime ]]; then - echo "=====> ALIVE <=====" - else - echo "=====> Considering host as DEAD due to file [RBD pool] exist, condition [suspectTimeDiff -lt 0] was satisfied and [latestUpdateTime -gt SuspectTime] has not been satisfied. <======" - fi - else - if [[ $latestUpdateTime -gt $lastUpdateTime ]]; then - echo "=====> ALIVE <=====" - else - echo "=====> Considering host as DEAD due to file [RBD pool] exist and conditions [suspectTimeDiff -lt 0] and [latestUpdateTime -gt SuspectTime] have not been satisfied. <======" - fi - fi + echo "=====> Considering host as DEAD due to [RBD '$PoolName' pool] Image Watcher does not exists <======" fi #Deleting Ceph keyring From d07541095b0a0270d3293267a20cdebfabe66304 Mon Sep 17 00:00:00 2001 From: sjyu Date: Mon, 21 Feb 2022 15:01:41 +0900 Subject: [PATCH 15/38] =?UTF-8?q?kvm=20ha=20>=20rbd=20=EC=8A=A4=ED=81=AC?= =?UTF-8?q?=EB=A6=BD=ED=8A=B8=20poolname=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kvm ha > rbd 스크립트 poolname 추가 --- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh | 10 +++++----- scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh index b4e1b9cc3c44..4b19fe5ecad9 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh @@ -88,14 +88,14 @@ delete_cephKeyring() { cretae_hbWatcher() { #Create HB RBD Image and watcher - status=$(rbd status hb-$HostIP -m $SourceHostIP -k $keyringFile) + status=$(rbd status hb-$HostIP --pool $PoolName -m $SourceHostIP -k $keyringFile) if [ $? == 2 ]; then - rbd create hb-$HostIP --size 1 -m $SourceHostIP -k $keyringFile - setsid sh -c 'exec rbd watch hb-'$HostIP' -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' + rbd create hb-$HostIP --size 1 --pool $PoolName + setsid sh -c 'exec rbd watch hb-'$HostIP' --pool $PoolName -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' fi if [ "$status" == "Watchers: none" ]; then - setsid sh -c 'exec rbd watch hb-'$HostIP' -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' + setsid sh -c 'exec rbd watch hb-'$HostIP' --pool $PoolName -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' fi return 0 @@ -103,7 +103,7 @@ cretae_hbWatcher() { check_hbWatcher() { #check the heart beat watcher - hb=$(rbd status hb-$HostIP -m $SourceHostIP -k $keyringFile) + hb=$(rbd status hb-$HostIP --pool $PoolName -m $SourceHostIP -k $keyringFile) if [ "$hb" == "Watchers: none" ]; then return 2 else diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh index 795e36d1359b..05db0b1c6d16 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh @@ -72,7 +72,7 @@ if [ ! -f $keyringFile ]; then fi # First check: heartbeat watcher -status=$(rbd status hb-$HostIP -m $SourceHostIP -k $keyringFile) +status=$(rbd status hb-$HostIP --pool $PoolName -m $SourceHostIP -k $keyringFile) if [ "$status" != "Watchers: none" ]; then echo "=====> ALIVE <=====" exit 0 @@ -86,7 +86,7 @@ fi # Second check: disk activity check statusFlag=true for UUID in $(echo $UUIDList | sed 's/,/ /g'); do - diskStatus=$(rbd status $UUID -m $SourceHostIP -k $keyringFile) + diskStatus=$(rbd status $UUID --pool $PoolName -m $SourceHostIP -k $keyringFile) if [ "$status" == "Watchers: none" ]; then statusFlag=false break From 70a9c5feb2490c584e03b36e868d7f133679293d Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Mon, 4 Apr 2022 17:20:28 +0900 Subject: [PATCH 16/38] =?UTF-8?q?=08hb,ac=20=EC=B2=B4=ED=81=AC=20=EC=8A=A4?= =?UTF-8?q?=ED=81=AC=EB=A6=BD=ED=8A=B8=EB=A5=BC=20Python=20librbd=EB=A1=9C?= =?UTF-8?q?=20=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hypervisor/kvm/resource/KVMHABase.java | 6 +- .../hypervisor/kvm/resource/KVMHAChecker.java | 70 ++++-- .../hypervisor/kvm/resource/KVMHAMonitor.java | 234 +++++++++++------- .../kvm/resource/KVMHAVMActivityChecker.java | 66 +++-- .../resource/LibvirtComputingResource.java | 47 +++- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py | 102 ++++++++ scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh | 136 ---------- .../vm/hypervisor/kvm/kvmvmactivity_rbd.py | 90 +++++++ .../vm/hypervisor/kvm/kvmvmactivity_rbd.sh | 107 -------- 9 files changed, 468 insertions(+), 390 deletions(-) create mode 100755 scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py delete mode 100755 scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh create mode 100755 scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py delete mode 100755 scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index e26262b9f530..74bc3597ef6a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -60,7 +60,7 @@ public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, Str public static class RbdStoragePool { String _poolUUID; - String _poolIp; + String _monHost; String _poolMountSourcePath; String _mountDestPath; PoolType _type; @@ -68,9 +68,9 @@ public static class RbdStoragePool { String _poolAuthSecret; String _poolSourceHost; - public RbdStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthUserName, String poolAuthSecret, String poolSourceHost) { + public RbdStoragePool(String poolUUID, String monHost, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthUserName, String poolAuthSecret, String poolSourceHost) { _poolUUID = poolUUID; - _poolIp = poolIp; + _monHost = monHost; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; _type = type; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index 292eb3c02ea0..c2c044491812 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -19,7 +19,9 @@ import java.util.List; import java.util.concurrent.Callable; import java.util.stream.Collectors; - +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import org.apache.log4j.Logger; import com.cloud.utils.script.OutputInterpreter; @@ -30,7 +32,7 @@ public class KVMHAChecker extends KVMHABase implements Callable { private List nfsStoragePools; private List rbdStoragePools; private String hostIp; - private long heartBeatCheckerTimeout = 360000; // 6 minutes + private long heartBeatCheckerTimeout = 600000; // 10 minutes public KVMHAChecker(List nfspools, List rbdpools, String host) { this.nfsStoragePools = nfspools; @@ -46,15 +48,18 @@ public KVMHAChecker(List nfspools, List rbdpools public Boolean checkingHeartBeat() { boolean validResult = false; - String hostAndPools = String.format("host IP [%s] in pools [%s]", hostIp, nfsStoragePools.stream().map(pool -> pool._poolIp).collect(Collectors.joining(", "))); + String hostAndPools = ""; s_logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools)); - for (NfsStoragePool pool : nfsStoragePools) { + for (NfsStoragePool nfspools : nfsStoragePools) { + hostAndPools = String.format("host IP [%s] in pools [%s]", hostIp, nfsStoragePools.stream().map(pool -> pool._poolIp).collect(Collectors.joining(", "))); + s_logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools)); + Script cmd = new Script(s_heartBeatPath, heartBeatCheckerTimeout, s_logger); - cmd.add("-i", pool._poolIp); - cmd.add("-p", pool._poolMountSourcePath); - cmd.add("-m", pool._mountDestPath); + cmd.add("-i", nfspools._poolIp); + cmd.add("-p", nfspools._poolMountSourcePath); + cmd.add("-m", nfspools._mountDestPath); cmd.add("-h", hostIp); cmd.add("-r"); cmd.add("-t", String.valueOf(_heartBeatUpdateFreq / 1000)); @@ -63,7 +68,7 @@ public Boolean checkingHeartBeat() { String parsedLine = parser.getLine(); s_logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, - pool._poolIp)); + nfspools._poolIp)); if (result == null && parsedLine.contains("DEAD")) { s_logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned [%s]. [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), @@ -73,24 +78,41 @@ public Boolean checkingHeartBeat() { } } - for (RbdStoragePool pool : rbdStoragePools) { - Script cmd = new Script(s_heartBeatPathRbd, heartBeatCheckerTimeout, s_logger); - cmd.add("-i", pool._poolSourceHost); - cmd.add("-p", pool._poolMountSourcePath); - cmd.add("-n", pool._poolAuthUserName); - cmd.add("-s", pool._poolAuthSecret); - cmd.add("-h", hostIp); - cmd.add("-r"); - OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); - String result = cmd.execute(parser); - String parsedLine = parser.getLine(); + for (RbdStoragePool rbdpools : rbdStoragePools) { + hostAndPools = String.format("host IP [%s] in pools [%s]", hostIp, rbdStoragePools.stream().map(pool -> pool._monHost).collect(Collectors.joining(", "))); + s_logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools)); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command().add("python3"); + processBuilder.command().add(s_heartBeatPathRbd); + processBuilder.command().add("-i"); + processBuilder.command().add(rbdpools._poolSourceHost); + processBuilder.command().add("-p"); + processBuilder.command().add(rbdpools._poolMountSourcePath); + processBuilder.command().add("-n"); + processBuilder.command().add(rbdpools._poolAuthUserName); + processBuilder.command().add("-s"); + processBuilder.command().add(rbdpools._poolAuthSecret); + processBuilder.command().add("-v"); + processBuilder.command().add(hostIp); + processBuilder.command().add("-r"); + processBuilder.command().add("r"); + Process process = null; + String parsedLine = ""; + try { + process = processBuilder.start(); + BufferedReader bfr = new BufferedReader(new InputStreamReader(process.getInputStream())); + parsedLine = bfr.readLine(); + } catch (IOException e) { + e.printStackTrace(); + } - s_logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, - pool._poolIp)); + s_logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", log: \"%s\", pool: \"%s\"}].", processBuilder.command().toString().replace(",", ""), parsedLine, + rbdpools._monHost)); - if (result == null && parsedLine.contains("DEAD")) { - s_logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned [%s]. [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), - result, parsedLine, hostIp)); + if (process != null && parsedLine.contains("DEAD")) { + s_logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned. [%s]. It may cause a shutdown of host IP [%s].", processBuilder.command().toString().replace(",", ""), + parsedLine, hostIp)); } else { validResult = true; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index af60ea7b129c..a8eef4a7eadf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -25,6 +25,9 @@ import org.libvirt.StoragePool; import org.libvirt.StoragePoolInfo.StoragePoolState; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -37,8 +40,6 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private static final Logger s_logger = Logger.getLogger(KVMHAMonitor.class); private final Map nfsstoragePool = new ConcurrentHashMap<>(); private final Map rbdstoragePool = new ConcurrentHashMap<>(); - private final NfsStoragePool nfsStoragePool = null; - private final RbdStoragePool rbdStoragePool = null; private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; @@ -117,13 +118,64 @@ public RbdStoragePool getRbdStoragePool(String uuid) { } protected void runHeartBeat() { - if (nfsstoragePool != null && !nfsstoragePool.isEmpty()) { + if(nfsstoragePool != null && !nfsstoragePool.isEmpty()) { synchronized (nfsstoragePool) { Set removedPools = new HashSet<>(); for (String uuid : nfsstoragePool.keySet()) { - NfsStoragePool nfsStoragePool = nfsstoragePool.get(uuid); - runHeartbeatToPool(nfsStoragePool, rbdStoragePool, uuid, removedPools); - continue; + NfsStoragePool primaryStoragePool = nfsstoragePool.get(uuid); + StoragePool storage; + try { + Connect conn = LibvirtConnection.getConnection(); + storage = conn.storagePoolLookupByUUIDString(uuid); + if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { + if (storage == null) { + s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + } else { + s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + } + + removedPools.add(uuid); + continue; + } + + s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + + } catch (LibvirtException e) { + s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + + if (e.toString().contains("pool not found")) { + s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + removedPools.add(uuid); + continue; + } + + } + + String result = null; + for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + Script cmd = createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); + result = cmd.execute(); + + s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); + + if (result != null) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); + try { + Thread.sleep(_heartBeatUpdateRetrySleep); + } catch (InterruptedException e) { + s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } + } else { + break; + } + + } + + if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + Script cmd = createHeartBeatCommand(primaryStoragePool, null, false); + result = cmd.execute(); + } } if (!removedPools.isEmpty()) { @@ -138,86 +190,81 @@ protected void runHeartBeat() { synchronized (rbdstoragePool) { Set removedPools = new HashSet<>(); for (String uuid : rbdstoragePool.keySet()) { - RbdStoragePool rbdStoragePool = rbdstoragePool.get(uuid); - runHeartbeatToPool(nfsStoragePool, rbdStoragePool, uuid, removedPools); - continue; - } + RbdStoragePool primaryStoragePool = rbdstoragePool.get(uuid); + StoragePool storage; + try { + Connect conn = LibvirtConnection.getConnection(); + storage = conn.storagePoolLookupByUUIDString(uuid); + if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { + if (storage == null) { + s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + } else { + s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + } + + removedPools.add(uuid); + continue; + } + + s_logger.debug(String.format("Found RBD storage pool [%s] in libvirt, continuing.", uuid)); + + } catch (LibvirtException e) { + s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + + if (e.toString().contains("pool not found")) { + s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + removedPools.add(uuid); + continue; + } - if (!removedPools.isEmpty()) { - for (String uuid : removedPools) { - removeRbdStoragePool(uuid); } - } - } - } - } - - private Set runHeartbeatToPool(NfsStoragePool nfsStoragePool, RbdStoragePool rbdStoragePool, String uuid, Set removedPools) { - StoragePool storage; - try { - Connect conn = LibvirtConnection.getConnection(); - storage = conn.storagePoolLookupByUUIDString(uuid); - if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - if (storage == null) { - s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); - } else { - s_logger.debug(String.format("Libvirt storage pool [%s] was found, but it is not running, removing it from HA list.", uuid)); - } - removedPools.add(uuid); - } - - s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); - - } catch (LibvirtException e) { - s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + String parsedLine = ""; + Process process = null; + for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + ProcessBuilder processBuilder = createRbdHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); + + try { + process = processBuilder.start(); + BufferedReader bfr = new BufferedReader(new InputStreamReader(process.getInputStream())); + parsedLine = bfr.readLine(); + } catch (IOException e) { + e.printStackTrace(); + } + + s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", processBuilder.command().toString().replace(",", ""), uuid, parsedLine)); + + if (process == null ) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, parsedLine, i, _heartBeatUpdateMaxTries)); + try { + Thread.sleep(_heartBeatUpdateRetrySleep); + } catch (InterruptedException e) { + s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } + } else { + break; + } - if (e.toString().contains("pool not found")) { - s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); - removedPools.add(uuid); - } - - } - - String result = null; - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { - Script cmd; - if (nfsStoragePool != null) { - cmd = createHeartBeatCommand(nfsStoragePool, hostPrivateIp, true); - result = cmd.execute(); - s_logger.debug(String.format("The command [%s], to the pool [%s], had the result [%s].", cmd.toString(), uuid, result)); - } else if (rbdStoragePool != null) { - cmd = createRbdHeartBeatCommand(rbdStoragePool, hostPrivateIp, true); - result = cmd.execute(); - s_logger.debug(String.format("The command [%s], to the pool [%s], had the result [%s].", cmd.toString(), uuid, result)); - } + } - if (result != null) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + if (process == null && rebootHostAndAlertManagementOnHeartbeatTimeout) { + s_logger.debug(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, parsedLine)); + ProcessBuilder processBuilder = createRbdHeartBeatCommand(primaryStoragePool, null, true); + try { + process = processBuilder.start(); + } catch (IOException e) { + e.printStackTrace(); + } + } } - } else { - break; - } - - } - if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); - Script cmd; - if (nfsStoragePool != null) { - cmd = createHeartBeatCommand(nfsStoragePool, null, false); - result = cmd.execute(); - } else if (rbdStoragePool != null) { - cmd = createRbdHeartBeatCommand(rbdStoragePool, null, false); - result = cmd.execute(); + if (!removedPools.isEmpty()) { + for (String uuid : removedPools) { + removeRbdStoragePool(uuid); + } + } } } - - return removedPools; } private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { @@ -237,22 +284,25 @@ private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String return cmd; } - private Script createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { - Script cmd = new Script(s_heartBeatPathRbd, _heartBeatUpdateTimeout, s_logger); - cmd.add("-i", primaryStoragePool._poolSourceHost); - cmd.add("-p", primaryStoragePool._poolMountSourcePath); - cmd.add("-n", primaryStoragePool._poolAuthUserName); - cmd.add("-s", primaryStoragePool._poolAuthSecret); - - if (hostValidation) { - cmd.add("-h", hostPrivateIp); - } - - if (!hostValidation) { - cmd.add("-c"); - } + private ProcessBuilder createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command().add("python3"); + processBuilder.command().add(s_heartBeatPathRbd); + processBuilder.command().add("-i"); + processBuilder.command().add(primaryStoragePool._poolSourceHost); + processBuilder.command().add("-p"); + processBuilder.command().add(primaryStoragePool._poolMountSourcePath); + processBuilder.command().add("-n"); + processBuilder.command().add(primaryStoragePool._poolAuthUserName); + processBuilder.command().add("-s"); + processBuilder.command().add(primaryStoragePool._poolAuthSecret); + + if (hostValidation) { + processBuilder.command().add("-v"); + processBuilder.command().add(hostPrivateIp); + } - return cmd; + return processBuilder; } @Override diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 8c6501acc9ab..18645608172f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -21,6 +21,9 @@ import com.cloud.storage.Storage.StoragePoolType; import org.apache.log4j.Logger; import org.joda.time.Duration; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.concurrent.Callable; @@ -29,7 +32,7 @@ public class KVMHAVMActivityChecker extends KVMHABase implements Callable params) th throw new ConfigurationException("Unable to find kvmheartbeat.sh"); } - _heartBeatPathRbd = Script.findScript(kvmScriptsDir, "kvmheartbeat_rbd.sh"); + _heartBeatPathRbd = Script.findScript(kvmScriptsDir, "kvmheartbeat_rbd.py"); if (_heartBeatPathRbd == null) { - throw new ConfigurationException("Unable to find kvmheartbeat_rbd.sh"); + throw new ConfigurationException("Unable to find kvmheartbeat_rbd.py"); } _createvmPath = Script.findScript(storageScriptsDir, "createvm.sh"); @@ -903,9 +903,9 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find kvmvmactivity.sh"); } - _vmActivityCheckPathRbd = Script.findScript(kvmScriptsDir, "kvmvmactivity_rbd.sh"); + _vmActivityCheckPathRbd = Script.findScript(kvmScriptsDir, "kvmvmactivity_rbd.py"); if (_vmActivityCheckPathRbd == null) { - throw new ConfigurationException("Unable to find kvmvmactivity_rbd.sh"); + throw new ConfigurationException("Unable to find kvmvmactivity_rbd.py"); } _createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); @@ -2870,8 +2870,14 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { We store the secret under the UUID of the pool, that's why we pass the pool's UUID as the authSecret */ - disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), - pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); + if (volume.getType() == Volume.Type.DATADISK && !(isWindowsTemplate && isUefiEnabled)) { + disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getUuid(), devId, diskBusTypeData, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); + } + else { + disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); + } } else if (pool.getType() == StoragePoolType.PowerFlex) { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData); } else if (pool.getType() == StoragePoolType.Gluster) { @@ -3039,7 +3045,7 @@ public boolean cleanupDisk(Map volumeToDisconnect) { public boolean cleanupDisk(final DiskDef disk) { final String path = disk.getDiskPath(); - if (path == null) { + if (org.apache.commons.lang.StringUtils.isBlank(path)) { s_logger.debug("Unable to clean up disk with null path (perhaps empty cdrom drive):" + disk); return false; } @@ -3930,6 +3936,7 @@ public VmStatsEntry getVmStat(final Connect conn, final String vmName) throws Li stats.setMemoryKBs(info.maxMem); stats.setTargetMemoryKBs(info.memory); stats.setIntFreeMemoryKBs(getMemoryFreeInKBs(dm)); + stats.setIntUsableMemoryKBs(getMemoryUsableInKBs(dm)); /* get cpu utilization */ VmStats oldStats = null; @@ -4038,7 +4045,29 @@ protected long getMemoryFreeInKBs(Domain dm) throws LibvirtException { if (ArrayUtils.isEmpty(mems)) { return NumberUtils.LONG_ZERO; } - return mems[0].getValue(); + //getMemoryFreeInKBs 메소드는 RSS 값을 출력, 값이 없는 경우 0 출력 + int length = mems.length; + for (int i = 0; i < length; i++) { + if (mems[i].getTag() == 7){ + return mems[i].getValue(); + } + } + return NumberUtils.LONG_ZERO; + } + + protected long getMemoryUsableInKBs(Domain dm) throws LibvirtException { + MemoryStatistic[] mems = dm.memoryStats(NUMMEMSTATS); + if (ArrayUtils.isEmpty(mems)) { + return NumberUtils.LONG_ZERO; + } + //getMemoryFreeInKBs 메소드는 USABLE 값을 출력, 값이 없는 경우 0 출력 + int length = mems.length; + for (int i = 0; i < length; i++) { + if (mems[i].getTag() == 8){ + return mems[i].getValue(); + } + } + return NumberUtils.LONG_ZERO; } private boolean canBridgeFirewall(final String prvNic) { diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py new file mode 100755 index 000000000000..a3aa61756fb1 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import os +import rbd +import rados +import time + +def createArgumentParser(): + parser=argparse.ArgumentParser(description='KVM RBD HA를 위한 인자') + parser.add_argument('-i', type=str, help='Source Host ip') + parser.add_argument('-p', type=str, help='rbd pool name') + parser.add_argument('-n', type=str, help='pool auth username') + parser.add_argument('-s', type=str, help='pool auth secret') + parser.add_argument('-v', type=str, help='host') + parser.add_argument('-r', type=str, help='create/read hb watcher') + + return parser + +confFile='/etc/ceph/ceph.conf' +keyringFolder='/run/cloudstack/agent' +keyringFile='keyring' + +def create_cephKeyring(): + # Create Ceph keyring for executing rbd commands + if not os.path.exists(keyringFolder): + os.makedirs(keyringFolder) + + if not os.path.isfile(keyringFolder+'/'+keyringFile): + f=open(keyringFolder+'/'+keyringFile,'w') + f.write('[client.'+args.n+']\n key='+args.s+'\n') + f.close() + +def delete_cephKeyring(): + # Delete Ceph keyring for executing rbd commands + if os.path.isfile(keyringFolder+'/'+keyringFile): + os.remove(keyringFolder+'/'+keyringFile) + +def watcher_list(): + # HB RBD Image List + cluster = rados.Rados(conffile=confFile) + try: + cluster.connect() + ioctx = cluster.open_ioctx(args.p) + try: + image = rbd.Image(ioctx, 'hb-'+args.v) + try: + watchers = list(image.watchers_list()) + return len(watchers) + finally: + image.close() + except rbd.ImageNotFound: + # Create HB RBD Image + create_rbdImage() + return 1 + finally: + ioctx.close() + finally: + cluster.shutdown() + +def create_rbdImage(): + # Create HB RBD Image + with rados.Rados(conffile=confFile) as cluster: + with cluster.open_ioctx(args.p) as ioctx: + rbd_inst = rbd.RBD() + size = 1 * 1024**3 # 1 GiB + rbd_inst.create(ioctx, 'hb-'+args.v, size) + with rbd.Image(ioctx, 'hb-'+args.v) as image: + data = b'foo' * 200 + image.write(data, 0) + +def create_hbWatcher(): + # Watch HB RBD Image + os.popen('setsid sh -c \"exec rbd watch hb-'+args.v+' --pool '+args.p+' -m '+args.i+' -k '+keyringFolder+'/'+keyringFile+' <> /dev/tty20 >&0 2>&1\"') + + +if __name__ == '__main__': + # parser 생성 + parser = createArgumentParser() + # 인자값 파싱 + args = parser.parse_args() + + create_cephKeyring() + + # Monitoring + if(args.r is None): + rs = watcher_list() + if rs == 1: + create_hbWatcher() + # Wait for watcher creation time + time.sleep(1) + + # Checker + else: + rs = watcher_list() + if rs == 2: + os.system('echo =====> ALIVE <=====') + else: + os.system('echo =====> Considering host as DEAD due to [hb-$HostIP] watcher that the host is seeing is not running. <======') + + delete_cephKeyring() diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh deleted file mode 100755 index 4b19fe5ecad9..000000000000 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -help() { - printf "Usage: $0 - -p rbd pool name - -n pool auth username - -s pool auth secret - -h host - -i source host ip - -r cretae/read hb watcher - -c cleanup" - exit 1 -} -#set -x -PoolName= -PoolAuthUserName= -PoolAuthSecret= -HostIP= -SourceHostIP= -rflag=0 -cflag=0 - -while getopts 'p:n:s:h:i:rc' OPTION -do - case $OPTION in - p) - PoolName="$OPTARG" - ;; - n) - PoolAuthUserName="$OPTARG" - ;; - s) - PoolAuthSecret="$OPTARG" - ;; - h) - HostIP="$OPTARG" - ;; - i) - SourceHostIP="$OPTARG" - ;; - r) - rflag=1 - ;; - c) - cflag=1 - ;; - *) - help - ;; - esac -done - -if [ -z "$PoolName" ]; then - exit 2 -fi - -keyringFile="/etc/cloudstack/agent/keyring" - -create_cephKeyring() { -#Creating Ceph keyring for executing rbd commands - if [ ! -f $keyringFile ]; then - echo -e "[client.$PoolAuthUserName]\n key=$PoolAuthSecret" > $keyringFile - fi -} - -delete_cephKeyring() { -#Deleting Ceph keyring - if [ -f $keyringFile ]; then - rm -rf $keyringFile - fi -} - -cretae_hbWatcher() { -#Create HB RBD Image and watcher - status=$(rbd status hb-$HostIP --pool $PoolName -m $SourceHostIP -k $keyringFile) - if [ $? == 2 ]; then - rbd create hb-$HostIP --size 1 --pool $PoolName - setsid sh -c 'exec rbd watch hb-'$HostIP' --pool $PoolName -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' - fi - - if [ "$status" == "Watchers: none" ]; then - setsid sh -c 'exec rbd watch hb-'$HostIP' --pool $PoolName -m '$SourceHostIP' -k '$keyringFile' <> /dev/tty20 >&0 2>&1' - fi - - return 0 -} - -check_hbWatcher() { -#check the heart beat watcher - hb=$(rbd status hb-$HostIP --pool $PoolName -m $SourceHostIP -k $keyringFile) - if [ "$hb" == "Watchers: none" ]; then - return 2 - else - return 0 - fi -} - -if [ "$rflag" == "1" ]; then - create_cephKeyring - check_hbWatcher - hb=$? - if [ "$hb" != "Watchers: none" ]; then - echo "=====> ALIVE <=====" - else - echo "=====> Considering host as DEAD due to [hb-$HostIP] watcher that the host is seeing is not running. <======" - fi - delete_cephKeyring - exit 0 -elif [ "$cflag" == "1" ]; then - /usr/bin/logger -t heartbeat "kvmheartbeat_rbd.sh reboots the system because there is no heartbeat watcher." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? -else - create_cephKeyring - cretae_hbWatcher - delete_cephKeyring - exit 0 -fi diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py new file mode 100755 index 000000000000..a173130bd7c6 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import os +import sys +import rbd +import rados + +def createArgumentParser(): + parser=argparse.ArgumentParser(description='KVM RBD HA를 위한 인자') + parser.add_argument('-i', help='Source Host ip') + parser.add_argument('-p', help='rbd pool name') + parser.add_argument('-n', help='pool auth username') + parser.add_argument('-s', help='pool auth secret') + parser.add_argument('-v', help='host') + parser.add_argument('-u', help='volume uuid list') + + return parser + +confFile='/etc/ceph/ceph.conf' +keyringFolder='/run/cloudstack/agent' +keyringFile='keyring.bin' + +def create_cephKeyring(): + # Create Ceph keyring for executing rbd commands + if not os.path.exists(keyringFolder): + os.makedirs(keyringFolder) + + if not os.path.isfile(keyringFolder+'/'+keyringFile): + f=open(keyringFolder+'/'+keyringFile,'w') + f.write('[client.'+args.n+']\n key='+args.s+'\n') + f.close() + +def delete_cephKeyring(): + # Delete Ceph keyring for executing rbd commands + if os.path.isfile(keyringFolder+'/'+keyringFile): + os.remove(keyringFolder+'/'+keyringFile) + +def watcher_list(i_name): + # HB RBD Image List + cluster = rados.Rados(conffile=confFile) + try: + cluster.connect() + ioctx = cluster.open_ioctx(args.p) + try: + image = rbd.Image(ioctx, i_name) + try: + watchers = list(image.watchers_list()) + return len(watchers) + finally: + image.close() + except rbd.ImageNotFound: + return 0 + finally: + ioctx.close() + finally: + cluster.shutdown() + +def check_diskActivity(): + for UUID in args.u.split(','): + ac_rs = watcher_list(UUID) + if ac_rs == 1: + return ac_rs + break + return 2 + + +if __name__ == '__main__': + # parser 생성 + parser = createArgumentParser() + # 인자값 파싱 + args = parser.parse_args() + + create_cephKeyring() + + # First check: heartbeat watcher + rs = watcher_list('hb-'+args.v) + if rs == 2: + print('=====> ALIVE <=====') + sys.exit(0) + + # Second check: disk activity check + ac_rs = check_diskActivity() + if ac_rs == 2: + print('=====> ALIVE <=====') + else: + print('=====> Considering host as DEAD due to [RBD '+args.p+' pool] Image Watcher does not exists <======') + + delete_cephKeyring() diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh deleted file mode 100755 index 05db0b1c6d16..000000000000 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -help() { - printf "Usage: $0 - -p rbd pool name - -n pool auth username - -s pool auth secret - -h host - -i source host ip - -u volume uuid list" - exit 1 -} -#set -x -PoolName= -PoolAuthUserName= -PoolAuthSecret= -HostIP= -SourceHostIP= -UUIDList= - -while getopts 'p:n:s:h:i:u:d:' OPTION -do - case $OPTION in - p) - PoolName="$OPTARG" - ;; - n) - PoolAuthUserName="$OPTARG" - ;; - s) - PoolAuthSecret="$OPTARG" - ;; - h) - HostIP="$OPTARG" - ;; - i) - SourceHostIP="$OPTARG" - ;; - u) - UUIDList="$OPTARG" - ;; - *) - help - ;; - esac -done - -if [ -z "$PoolName" ]; then - exit 2 -fi - -#Creating Ceph keyring for executing rbd commands -keyringFile="/etc/cloudstack/agent/keyring.bin" - -if [ ! -f $keyringFile ]; then - echo -e "[client.$PoolAuthUserName]\n key=$PoolAuthSecret" > $keyringFile -fi - -# First check: heartbeat watcher -status=$(rbd status hb-$HostIP --pool $PoolName -m $SourceHostIP -k $keyringFile) -if [ "$status" != "Watchers: none" ]; then - echo "=====> ALIVE <=====" - exit 0 -fi - -if [ -z "$UUIDList" ]; then - echo "=====> Considering host as DEAD due to empty UUIDList <======" - exit 0 -fi - -# Second check: disk activity check -statusFlag=true -for UUID in $(echo $UUIDList | sed 's/,/ /g'); do - diskStatus=$(rbd status $UUID --pool $PoolName -m $SourceHostIP -k $keyringFile) - if [ "$status" == "Watchers: none" ]; then - statusFlag=false - break - fi -done - -if [ statusFlag == "true" ]; then - echo "=====> ALIVE <=====" -else - echo "=====> Considering host as DEAD due to [RBD '$PoolName' pool] Image Watcher does not exists <======" -fi - -#Deleting Ceph keyring -if [ -f $keyringFile ]; then - rm -rf $keyringFile -fi - -exit 0 \ No newline at end of file From 4b149dad15f7fb376184e5b987f9318a52eeaf99 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 5 Apr 2022 10:37:31 +0900 Subject: [PATCH 17/38] =?UTF-8?q?=EC=86=8C=EC=8A=A4=EB=B3=B5=EC=9B=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resource/LibvirtComputingResource.java | 39 +++---------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 74a62aacbe3a..baf38f846c60 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -304,7 +304,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private long _hvVersion; private Duration _timeout; - private static final int NUMMEMSTATS =13; + private static final int NUMMEMSTATS =2; private KVMHAMonitor _monitor; public static final String SSHKEYSPATH = "/root/.ssh"; @@ -2870,14 +2870,8 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { We store the secret under the UUID of the pool, that's why we pass the pool's UUID as the authSecret */ - if (volume.getType() == Volume.Type.DATADISK && !(isWindowsTemplate && isUefiEnabled)) { - disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), - pool.getUuid(), devId, diskBusTypeData, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); - } - else { - disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), - pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); - } + disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); } else if (pool.getType() == StoragePoolType.PowerFlex) { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData); } else if (pool.getType() == StoragePoolType.Gluster) { @@ -3045,7 +3039,7 @@ public boolean cleanupDisk(Map volumeToDisconnect) { public boolean cleanupDisk(final DiskDef disk) { final String path = disk.getDiskPath(); - if (org.apache.commons.lang.StringUtils.isBlank(path)) { + if (path == null) { s_logger.debug("Unable to clean up disk with null path (perhaps empty cdrom drive):" + disk); return false; } @@ -3936,7 +3930,6 @@ public VmStatsEntry getVmStat(final Connect conn, final String vmName) throws Li stats.setMemoryKBs(info.maxMem); stats.setTargetMemoryKBs(info.memory); stats.setIntFreeMemoryKBs(getMemoryFreeInKBs(dm)); - stats.setIntUsableMemoryKBs(getMemoryUsableInKBs(dm)); /* get cpu utilization */ VmStats oldStats = null; @@ -4045,29 +4038,7 @@ protected long getMemoryFreeInKBs(Domain dm) throws LibvirtException { if (ArrayUtils.isEmpty(mems)) { return NumberUtils.LONG_ZERO; } - //getMemoryFreeInKBs 메소드는 RSS 값을 출력, 값이 없는 경우 0 출력 - int length = mems.length; - for (int i = 0; i < length; i++) { - if (mems[i].getTag() == 7){ - return mems[i].getValue(); - } - } - return NumberUtils.LONG_ZERO; - } - - protected long getMemoryUsableInKBs(Domain dm) throws LibvirtException { - MemoryStatistic[] mems = dm.memoryStats(NUMMEMSTATS); - if (ArrayUtils.isEmpty(mems)) { - return NumberUtils.LONG_ZERO; - } - //getMemoryFreeInKBs 메소드는 USABLE 값을 출력, 값이 없는 경우 0 출력 - int length = mems.length; - for (int i = 0; i < length; i++) { - if (mems[i].getTag() == 8){ - return mems[i].getValue(); - } - } - return NumberUtils.LONG_ZERO; + return mems[0].getValue(); } private boolean canBridgeFirewall(final String prvNic) { From 25f51a8905a0ff71be1978ab4cfa3de642806c33 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 5 Apr 2022 11:23:54 +0900 Subject: [PATCH 18/38] =?UTF-8?q?heartbeatUpdateMaxTries=20=EC=88=98?= =?UTF-8?q?=EC=A0=95,=20synchronized=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hypervisor/kvm/resource/KVMHABase.java | 2 +- .../hypervisor/kvm/resource/KVMHAChecker.java | 6 +- .../hypervisor/kvm/resource/KVMHAMonitor.java | 299 +++++++----------- .../kvm/resource/KVMHAVMActivityChecker.java | 2 +- 4 files changed, 122 insertions(+), 187 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 74bc3597ef6a..cdcbf60f61b4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -35,7 +35,7 @@ public class KVMHABase { protected static String s_heartBeatPathRbd; protected long _heartBeatUpdateTimeout = 60000; protected long _heartBeatUpdateFreq = 60000; - protected long _heartBeatUpdateMaxTries = 10; + protected long _heartBeatUpdateMaxTries = 5; protected long _heartBeatUpdateRetrySleep = 10000; public static enum PoolType { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index c2c044491812..b64550a6ec4a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -32,7 +32,7 @@ public class KVMHAChecker extends KVMHABase implements Callable { private List nfsStoragePools; private List rbdStoragePools; private String hostIp; - private long heartBeatCheckerTimeout = 600000; // 10 minutes + private long heartBeatCheckerTimeout = 360000; // 6 minutes public KVMHAChecker(List nfspools, List rbdpools, String host) { this.nfsStoragePools = nfspools; @@ -107,11 +107,11 @@ public Boolean checkingHeartBeat() { e.printStackTrace(); } - s_logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", log: \"%s\", pool: \"%s\"}].", processBuilder.command().toString().replace(",", ""), parsedLine, + s_logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", log: \"%s\", pool: \"%s\"}].", processBuilder.command().toString(), parsedLine, rbdpools._monHost)); if (process != null && parsedLine.contains("DEAD")) { - s_logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned. [%s]. It may cause a shutdown of host IP [%s].", processBuilder.command().toString().replace(",", ""), + s_logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned. [%s]. It may cause a shutdown of host IP [%s].", processBuilder.command().toString(), parsedLine, hostIp)); } else { validResult = true; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index a8eef4a7eadf..53c9f03c3c06 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -40,6 +40,8 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private static final Logger s_logger = Logger.getLogger(KVMHAMonitor.class); private final Map nfsstoragePool = new ConcurrentHashMap<>(); private final Map rbdstoragePool = new ConcurrentHashMap<>(); + private final NfsStoragePool nfsStoragePool = null; + private final RbdStoragePool rbdStoragePool = null; private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; @@ -62,206 +64,139 @@ private static synchronized void configureHeartBeatPath(String scriptPath, Strin } public void addStoragePool(NfsStoragePool pool) { - synchronized (nfsstoragePool) { - nfsstoragePool.put(pool._poolUUID, pool); - } + nfsstoragePool.put(pool._poolUUID, pool); } public void addStoragePool(RbdStoragePool pool) { - synchronized (rbdstoragePool) { - rbdstoragePool.put(pool._poolUUID, pool); - } + rbdstoragePool.put(pool._poolUUID, pool); } public void removeStoragePool(String uuid) { - synchronized (nfsstoragePool) { - NfsStoragePool pool = nfsstoragePool.get(uuid); - if (pool != null) { - Script.runSimpleBashScript("umount " + pool._mountDestPath); - nfsstoragePool.remove(uuid); - } + NfsStoragePool pool = nfsstoragePool.get(uuid); + if (pool != null) { + Script.runSimpleBashScript("umount " + pool._mountDestPath); + nfsstoragePool.remove(uuid); } } public void removeRbdStoragePool(String uuid) { - synchronized (rbdstoragePool) { - RbdStoragePool pool = rbdstoragePool.get(uuid); - if (pool != null) { - Script.runSimpleBashScript("umount " + pool._mountDestPath); - rbdstoragePool.remove(uuid); - } + RbdStoragePool pool = rbdstoragePool.get(uuid); + if (pool != null) { + Script.runSimpleBashScript("umount " + pool._mountDestPath); + rbdstoragePool.remove(uuid); } } public List getStoragePools() { - synchronized (nfsstoragePool) { - return new ArrayList<>(nfsstoragePool.values()); - } + return new ArrayList<>(nfsstoragePool.values()); } public List getRbdStoragePools() { - synchronized (rbdstoragePool) { - return new ArrayList<>(rbdstoragePool.values()); - } + return new ArrayList<>(rbdstoragePool.values()); } public NfsStoragePool getStoragePool(String uuid) { - synchronized (nfsstoragePool) { - return nfsstoragePool.get(uuid); - } + return nfsstoragePool.get(uuid); } public RbdStoragePool getRbdStoragePool(String uuid) { - synchronized (rbdstoragePool) { - return rbdstoragePool.get(uuid); - } + return rbdstoragePool.get(uuid); } protected void runHeartBeat() { - if(nfsstoragePool != null && !nfsstoragePool.isEmpty()) { - synchronized (nfsstoragePool) { - Set removedPools = new HashSet<>(); - for (String uuid : nfsstoragePool.keySet()) { - NfsStoragePool primaryStoragePool = nfsstoragePool.get(uuid); - StoragePool storage; - try { - Connect conn = LibvirtConnection.getConnection(); - storage = conn.storagePoolLookupByUUIDString(uuid); - if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - if (storage == null) { - s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); - } else { - s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); - } - - removedPools.add(uuid); - continue; - } - - s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); - - } catch (LibvirtException e) { - s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); - - if (e.toString().contains("pool not found")) { - s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); - removedPools.add(uuid); - continue; - } - - } - - String result = null; - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { - Script cmd = createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - result = cmd.execute(); - - s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", cmd.toString(), uuid, result)); - - if (result != null) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { - break; - } - - } - - if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); - Script cmd = createHeartBeatCommand(primaryStoragePool, null, false); - result = cmd.execute(); - } - } + Set removedPools = new HashSet<>(); + if (nfsstoragePool != null && !nfsstoragePool.isEmpty()) { + for (String uuid : nfsstoragePool.keySet()) { + NfsStoragePool nfsStoragePool = nfsstoragePool.get(uuid); + runHeartbeatToPool(nfsStoragePool, rbdStoragePool, uuid, removedPools); + } + } + + if (rbdstoragePool != null && !rbdstoragePool.isEmpty()) { + for (String uuid : rbdstoragePool.keySet()) { + RbdStoragePool rbdStoragePool = rbdstoragePool.get(uuid); + runHeartbeatToPool(nfsStoragePool, rbdStoragePool, uuid, removedPools); + } + } + + if (!removedPools.isEmpty()) { + for (String uuid : removedPools) { + removeStoragePool(uuid); + } + } + } - if (!removedPools.isEmpty()) { - for (String uuid : removedPools) { - removeStoragePool(uuid); - } + private void runHeartbeatToPool(NfsStoragePool nfsStoragePool, RbdStoragePool rbdStoragePool, String uuid, Set removedPools) { + StoragePool storage; + try { + Connect conn = LibvirtConnection.getConnection(); + storage = conn.storagePoolLookupByUUIDString(uuid); + if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { + if (storage == null) { + s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + } else { + s_logger.debug(String.format("Libvirt storage pool [%s] was found, but it is not running, removing it from the HA list.", uuid)); } + + removedPools.add(uuid); + } + + s_logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + + } catch (LibvirtException e) { + s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + + if (e.toString().contains("pool not found")) { + s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + removedPools.add(uuid); } + } - if (rbdstoragePool != null && !rbdstoragePool.isEmpty()) { - synchronized (rbdstoragePool) { - Set removedPools = new HashSet<>(); - for (String uuid : rbdstoragePool.keySet()) { - RbdStoragePool primaryStoragePool = rbdstoragePool.get(uuid); - StoragePool storage; - try { - Connect conn = LibvirtConnection.getConnection(); - storage = conn.storagePoolLookupByUUIDString(uuid); - if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { - if (storage == null) { - s_logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); - } else { - s_logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); - } - - removedPools.add(uuid); - continue; - } - - s_logger.debug(String.format("Found RBD storage pool [%s] in libvirt, continuing.", uuid)); - - } catch (LibvirtException e) { - s_logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); - - if (e.toString().contains("pool not found")) { - s_logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); - removedPools.add(uuid); - continue; - } - - } - - String parsedLine = ""; - Process process = null; - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { - ProcessBuilder processBuilder = createRbdHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - - try { - process = processBuilder.start(); - BufferedReader bfr = new BufferedReader(new InputStreamReader(process.getInputStream())); - parsedLine = bfr.readLine(); - } catch (IOException e) { - e.printStackTrace(); - } - - s_logger.debug(String.format("The command (%s), to the pool [%s], has the result [%s].", processBuilder.command().toString().replace(",", ""), uuid, parsedLine)); - - if (process == null ) { - s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, parsedLine, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { - break; - } - - } - - if (process == null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - s_logger.debug(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, parsedLine)); - ProcessBuilder processBuilder = createRbdHeartBeatCommand(primaryStoragePool, null, true); - try { - process = processBuilder.start(); - } catch (IOException e) { - e.printStackTrace(); - } - } + String result = null; + Process process = null; + for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + if (nfsStoragePool != null) { + Script cmd = createHeartBeatCommand(nfsStoragePool, hostPrivateIp, true); + result = cmd.execute(); + s_logger.debug(String.format("The command [%s], to the pool [%s], had the result [%s].", cmd.toString(), uuid, result)); + } else if (rbdStoragePool != null) { + ProcessBuilder processBuilder = createRbdHeartBeatCommand(rbdStoragePool, hostPrivateIp, true); + try { + process = processBuilder.start(); + BufferedReader bfr = new BufferedReader(new InputStreamReader(process.getInputStream())); + result = bfr.readLine(); + } catch (IOException e) { + e.printStackTrace(); } + s_logger.debug(String.format("The command [%s], to the pool [%s], had the result [%s].", processBuilder.command().toString(), uuid, result)); + } + if (result != null) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); + try { + Thread.sleep(_heartBeatUpdateRetrySleep); + } catch (InterruptedException e) { + s_logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } + } else { + break; + } + + } - if (!removedPools.isEmpty()) { - for (String uuid : removedPools) { - removeRbdStoragePool(uuid); - } + if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { + s_logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + if (nfsStoragePool != null) { + Script cmd = createHeartBeatCommand(nfsStoragePool, null, false); + result = cmd.execute(); + } else if (rbdStoragePool != null) { + ProcessBuilder processBuilder = createRbdHeartBeatCommand(rbdStoragePool, null, false); + try { + process = processBuilder.start(); + BufferedReader bfr = new BufferedReader(new InputStreamReader(process.getInputStream())); + result = bfr.readLine(); + } catch (IOException e) { + e.printStackTrace(); } } } @@ -286,21 +221,21 @@ private Script createHeartBeatCommand(NfsStoragePool primaryStoragePool, String private ProcessBuilder createRbdHeartBeatCommand(RbdStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.command().add("python3"); - processBuilder.command().add(s_heartBeatPathRbd); - processBuilder.command().add("-i"); - processBuilder.command().add(primaryStoragePool._poolSourceHost); - processBuilder.command().add("-p"); - processBuilder.command().add(primaryStoragePool._poolMountSourcePath); - processBuilder.command().add("-n"); - processBuilder.command().add(primaryStoragePool._poolAuthUserName); - processBuilder.command().add("-s"); - processBuilder.command().add(primaryStoragePool._poolAuthSecret); - - if (hostValidation) { - processBuilder.command().add("-v"); - processBuilder.command().add(hostPrivateIp); - } + processBuilder.command().add("python3"); + processBuilder.command().add(s_heartBeatPathRbd); + processBuilder.command().add("-i"); + processBuilder.command().add(primaryStoragePool._poolSourceHost); + processBuilder.command().add("-p"); + processBuilder.command().add(primaryStoragePool._poolMountSourcePath); + processBuilder.command().add("-n"); + processBuilder.command().add(primaryStoragePool._poolAuthUserName); + processBuilder.command().add("-s"); + processBuilder.command().add(primaryStoragePool._poolAuthSecret); + + if (hostValidation) { + processBuilder.command().add("-v"); + processBuilder.command().add(hostPrivateIp); + } return processBuilder; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index 18645608172f..59a1bb75147f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -87,7 +87,7 @@ public Boolean checkingHeartBeat() { processBuilder.command().add(hostIp); processBuilder.command().add("-u"); processBuilder.command().add(volumeUuidList); - command = processBuilder.command().toString().replace(",", ""); + command = processBuilder.command().toString(); Process process = null; try { From 988551cb687d8de6fb04cba80cd4cd974f143923 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Wed, 6 Apr 2022 11:38:04 +0900 Subject: [PATCH 19/38] KVMHAMonitor --- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py | 6 +++--- scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py index a3aa61756fb1..6973308f62d0 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py @@ -8,7 +8,7 @@ import time def createArgumentParser(): - parser=argparse.ArgumentParser(description='KVM RBD HA를 위한 인자') + parser=argparse.ArgumentParser(description='Argument for KVM RBD HA') parser.add_argument('-i', type=str, help='Source Host ip') parser.add_argument('-p', type=str, help='rbd pool name') parser.add_argument('-n', type=str, help='pool auth username') @@ -76,9 +76,9 @@ def create_hbWatcher(): if __name__ == '__main__': - # parser 생성 + # create parser parser = createArgumentParser() - # 인자값 파싱 + # argument parsing args = parser.parse_args() create_cephKeyring() diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py index a173130bd7c6..42cfac911205 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py @@ -8,7 +8,7 @@ import rados def createArgumentParser(): - parser=argparse.ArgumentParser(description='KVM RBD HA를 위한 인자') + parser=argparse.ArgumentParser(description='Argument for KVM RBD HA') parser.add_argument('-i', help='Source Host ip') parser.add_argument('-p', help='rbd pool name') parser.add_argument('-n', help='pool auth username') @@ -67,9 +67,9 @@ def check_diskActivity(): if __name__ == '__main__': - # parser 생성 + # create parser parser = createArgumentParser() - # 인자값 파싱 + # argument parsing args = parser.parse_args() create_cephKeyring() From 54bf917841dfd0d3556b9159cadeaf6d60a8b6e4 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Thu, 7 Apr 2022 12:48:29 +0900 Subject: [PATCH 20/38] librbd setting, mon_host to conffile --- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py | 5 ++--- scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py index 6973308f62d0..32cb5ab37928 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py @@ -18,7 +18,6 @@ def createArgumentParser(): return parser -confFile='/etc/ceph/ceph.conf' keyringFolder='/run/cloudstack/agent' keyringFile='keyring' @@ -39,7 +38,7 @@ def delete_cephKeyring(): def watcher_list(): # HB RBD Image List - cluster = rados.Rados(conffile=confFile) + cluster = rados.Rados(conf={'mon_host': args.i, 'keyring': keyringFolder+"/"+keyringFile}) try: cluster.connect() ioctx = cluster.open_ioctx(args.p) @@ -61,7 +60,7 @@ def watcher_list(): def create_rbdImage(): # Create HB RBD Image - with rados.Rados(conffile=confFile) as cluster: + with rados.Rados(conf={'mon_host': args.i, 'keyring': keyringFolder+"/"+keyringFile}) as cluster: with cluster.open_ioctx(args.p) as ioctx: rbd_inst = rbd.RBD() size = 1 * 1024**3 # 1 GiB diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py index 42cfac911205..bacda0a61690 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py @@ -18,7 +18,6 @@ def createArgumentParser(): return parser -confFile='/etc/ceph/ceph.conf' keyringFolder='/run/cloudstack/agent' keyringFile='keyring.bin' @@ -39,7 +38,7 @@ def delete_cephKeyring(): def watcher_list(i_name): # HB RBD Image List - cluster = rados.Rados(conffile=confFile) + cluster = rados.Rados(conf={'mon_host': args.i, 'keyring': keyringFolder+"/"+keyringFile}) try: cluster.connect() ioctx = cluster.open_ioctx(args.p) From 582e29306232c391f7f60d7129136708787a1693 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Mon, 11 Apr 2022 16:03:11 +0900 Subject: [PATCH 21/38] =?UTF-8?q?ASF=20=EB=9D=BC=EC=9D=B4=EC=84=BC?= =?UTF-8?q?=EC=8A=A4=20=EC=A0=95=EB=B3=B4=EB=A5=BC=20=EC=9E=91=EC=84=B1=20?= =?UTF-8?q?=EB=B0=8F=20=EA=B3=B5=EB=B0=B1=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py | 26 ++++++++++++++----- .../vm/hypervisor/kvm/kvmvmactivity_rbd.py | 20 +++++++++++--- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py index 32cb5ab37928..7644f582deef 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py @@ -1,5 +1,19 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. import argparse import os @@ -23,7 +37,7 @@ def createArgumentParser(): def create_cephKeyring(): # Create Ceph keyring for executing rbd commands - if not os.path.exists(keyringFolder): + if not os.path.exists(keyringFolder): os.makedirs(keyringFolder) if not os.path.isfile(keyringFolder+'/'+keyringFile): @@ -70,7 +84,7 @@ def create_rbdImage(): image.write(data, 0) def create_hbWatcher(): - # Watch HB RBD Image + # Watch HB RBD Image os.popen('setsid sh -c \"exec rbd watch hb-'+args.v+' --pool '+args.p+' -m '+args.i+' -k '+keyringFolder+'/'+keyringFile+' <> /dev/tty20 >&0 2>&1\"') @@ -83,11 +97,11 @@ def create_hbWatcher(): create_cephKeyring() # Monitoring - if(args.r is None): + if(args.r is None): rs = watcher_list() if rs == 1: create_hbWatcher() - # Wait for watcher creation time + # Wait for watcher creation time time.sleep(1) # Checker diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py index bacda0a61690..33846c19a592 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py @@ -1,5 +1,19 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. import argparse import os @@ -23,7 +37,7 @@ def createArgumentParser(): def create_cephKeyring(): # Create Ceph keyring for executing rbd commands - if not os.path.exists(keyringFolder): + if not os.path.exists(keyringFolder): os.makedirs(keyringFolder) if not os.path.isfile(keyringFolder+'/'+keyringFile): From cebc6b52c17291dc35a8def4487f59ec09711fee Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 3 May 2022 09:10:37 +0900 Subject: [PATCH 22/38] Changed so that ha occurs when oobm is off --- .../main/java/com/cloud/ha/KVMInvestigator.java | 16 ++++++++-------- .../apache/cloudstack/kvm/ha/KVMHAProvider.java | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java index a76b56a1a4de..816744b067c5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java @@ -78,25 +78,25 @@ public Status isAgentAlive(Host agent) { } List clusterPools = _storagePoolDao.listPoolsByCluster(agent.getClusterId()); - boolean hasNfs = false; + boolean hasHaPool = false; for (StoragePoolVO pool : clusterPools) { - if (pool.getPoolType() == StoragePoolType.NetworkFilesystem) { - hasNfs = true; + if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.RBD) { + hasHaPool = true; break; } } - if (!hasNfs) { + if (!hasHaPool) { List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType()); for (StoragePoolVO pool : zonePools) { - if (pool.getPoolType() == StoragePoolType.NetworkFilesystem) { - hasNfs = true; + if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.RBD) { + hasHaPool = true; break; } } } - if (!hasNfs) { + if (!hasHaPool) { s_logger.warn( - "Agent investigation was requested on host " + agent + ", but host does not support investigation because it has no NFS storage. Skipping investigation."); + "Agent investigation was requested on host " + agent + ", but host does not support investigation because it has no NFS, RBD storage. Skipping investigation."); return Status.Disconnected; } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index 5399fd23a1cd..4f47a1d5b179 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -72,8 +72,9 @@ public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACh public boolean recover(Host r) throws HARecoveryException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ - final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); - return resp.getSuccess(); + // final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); + // return resp.getSuccess(); + return false; // Changed so that ha occurs when oobm is off } else { LOG.warn("OOBM recover operation failed for the host " + r.getName()); return false; From 3b59683370ac21233e3a8e44231ea8a1d30cc81e Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 10 May 2022 16:36:02 +0900 Subject: [PATCH 23/38] Changed so that ha occurs when oobm is off --- .../cloudstack/kvm/ha/KVMHAProvider.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index 4f47a1d5b179..129fbf21d4c5 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -33,6 +33,9 @@ import org.apache.cloudstack.ha.provider.host.HAAbstractHostProvider; import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerOperation; import org.apache.cloudstack.outofbandmanagement.OutOfBandManagementService; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerState; +import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; +import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; import org.apache.log4j.Logger; import org.joda.time.DateTime; @@ -46,6 +49,8 @@ public final class KVMHAProvider extends HAAbstractHostProvider implements HAPro protected KVMHostActivityChecker hostActivityChecker; @Inject protected OutOfBandManagementService outOfBandManagementService; + @Inject + private OutOfBandManagementDao outOfBandManagementDao; @Override public boolean isEligible(final Host host) { @@ -72,9 +77,13 @@ public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACh public boolean recover(Host r) throws HARecoveryException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ - // final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); - // return resp.getSuccess(); - return false; // Changed so that ha occurs when oobm is off + final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); + if(oobm.getPowerState() == PowerState.Off){ + return false; // Changed so that ha occurs when oobm is off + }else{ + final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); + return resp.getSuccess(); + } } else { LOG.warn("OOBM recover operation failed for the host " + r.getName()); return false; @@ -89,8 +98,13 @@ public boolean recover(Host r) throws HARecoveryException { public boolean fence(Host r) throws HAFenceException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ - final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null); - return resp.getSuccess(); + final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); + if (oobm.getPowerState() == PowerState.Unknown){ + return true; + } else { + final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null); + return resp.getSuccess(); + } } else { LOG.warn("OOBM fence operation failed for this host " + r.getName()); return false; From 85214896517e679655b169a9318af2c0dca545fd Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 10 May 2022 16:46:14 +0900 Subject: [PATCH 24/38] =?UTF-8?q?OOBM=20PowerState.off=EC=8B=9C=20?= =?UTF-8?q?=EB=A1=9C=EA=B7=B8=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index 129fbf21d4c5..8c35a9fef7c9 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -79,6 +79,7 @@ public boolean recover(Host r) throws HARecoveryException { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); if(oobm.getPowerState() == PowerState.Off){ + LOG.warn("OOBM recover operation failed for the host " + r.getName() + " already OFF"); return false; // Changed so that ha occurs when oobm is off }else{ final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); From 2c046cffb55a637dfd97efbfa98000f8dfccd012 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 10 May 2022 17:52:03 +0900 Subject: [PATCH 25/38] =?UTF-8?q?=EC=A3=BC=EC=84=9D=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index 8c35a9fef7c9..cb92b053c96b 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -80,7 +80,7 @@ public boolean recover(Host r) throws HARecoveryException { final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); if(oobm.getPowerState() == PowerState.Off){ LOG.warn("OOBM recover operation failed for the host " + r.getName() + " already OFF"); - return false; // Changed so that ha occurs when oobm is off + return false; }else{ final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); return resp.getSuccess(); From 8534981e593841b66cf0d7ca085dc9941eb87648 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Mon, 30 May 2022 12:56:08 +0900 Subject: [PATCH 26/38] INVALIDATE CACHE Test --- .../cloudstack/kvm/ha/KVMHAProvider.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index cb92b053c96b..9f4794bc5ce4 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -21,6 +21,14 @@ import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.response.OutOfBandManagementResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -36,8 +44,13 @@ import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerState; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; + import org.apache.log4j.Logger; import org.joda.time.DateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import javax.inject.Inject; import java.security.InvalidParameterException; @@ -51,6 +64,12 @@ public final class KVMHAProvider extends HAAbstractHostProvider implements HAPro protected OutOfBandManagementService outOfBandManagementService; @Inject private OutOfBandManagementDao outOfBandManagementDao; + @Inject + private VolumeDao volumeDao; + @Inject + private VMInstanceDao vmInstanceDao; + @Inject + private PrimaryDataStoreDao storagePool; @Override public boolean isEligible(final Host host) { @@ -98,6 +117,16 @@ public boolean recover(Host r) throws HARecoveryException { @Override public boolean fence(Host r) throws HAFenceException { try { + //INVALIDATE CACHE Test + HashMap> poolVolMap = getVolumeUuidOnHost(r); + for (StoragePool pool : poolVolMap.keySet()) { + List volume_list = poolVolMap.get(pool); + LOG.warn("=====================KVMHAProvider.java===="); + LOG.warn("pool = "+pool); + LOG.warn("volume_list = "+volume_list); + LOG.warn("========================="); + } + if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); if (oobm.getPowerState() == PowerState.Unknown){ @@ -116,6 +145,31 @@ public boolean fence(Host r) throws HAFenceException { } } + private HashMap> getVolumeUuidOnHost(Host r) { + List vm_list = vmInstanceDao.listByHostId(r.getId()); + List volume_list = new ArrayList(); + for (VirtualMachine vm : vm_list) { + LOG.debug(String.format("Retrieving volumes of VM [%s]...", vm.getId())); + List vm_volume_list = volumeDao.findByInstance(vm.getId()); + volume_list.addAll(vm_volume_list); + } + + HashMap> poolVolMap = new HashMap>(); + for (Volume vol : volume_list) { + LOG.debug(String.format("Retrieving storage pool [%s] of volume [%s]...", vol.getPoolId(), vol.getId())); + StoragePool sp = storagePool.findById(vol.getPoolId()); + if (!poolVolMap.containsKey(sp)) { + List list = new ArrayList(); + list.add(vol); + + poolVolMap.put(sp, list); + } else { + poolVolMap.get(sp).add(vol); + } + } + return poolVolMap; + } + @Override public HAResource.ResourceSubType resourceSubType() { return HAResource.ResourceSubType.KVM; From 0db6133926e77821871dbc069b35c2d4ecda8108 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Mon, 30 May 2022 13:21:03 +0900 Subject: [PATCH 27/38] delete import --- .../java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index 9f4794bc5ce4..5aac090c6b58 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -21,7 +21,6 @@ import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -125,7 +124,7 @@ public boolean fence(Host r) throws HAFenceException { LOG.warn("pool = "+pool); LOG.warn("volume_list = "+volume_list); LOG.warn("========================="); - } + } if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); @@ -169,7 +168,7 @@ private HashMap> getVolumeUuidOnHost(Host r) { } return poolVolMap; } - + @Override public HAResource.ResourceSubType resourceSubType() { return HAResource.ResourceSubType.KVM; From b29928c0f9e2a1920cc2a9009bae5b450dac9ebe Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Mon, 30 May 2022 13:45:45 +0900 Subject: [PATCH 28/38] =?UTF-8?q?=EC=86=8C=EC=8A=A4=EC=A0=95=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cloudstack/kvm/ha/KVMHAProvider.java | 45 ++----------------- .../kvm/ha/KVMHostActivityChecker.java | 2 +- 2 files changed, 4 insertions(+), 43 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index 5aac090c6b58..fb278a4f2d3f 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -23,11 +23,6 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.response.OutOfBandManagementResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -43,16 +38,13 @@ import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement.PowerState; import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao; import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; - import org.apache.log4j.Logger; import org.joda.time.DateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import javax.inject.Inject; import java.security.InvalidParameterException; +import java.util.HashMap; +import java.util.List; public final class KVMHAProvider extends HAAbstractHostProvider implements HAProvider, Configurable { private final static Logger LOG = Logger.getLogger(KVMHAProvider.class); @@ -63,12 +55,6 @@ public final class KVMHAProvider extends HAAbstractHostProvider implements HAPro protected OutOfBandManagementService outOfBandManagementService; @Inject private OutOfBandManagementDao outOfBandManagementDao; - @Inject - private VolumeDao volumeDao; - @Inject - private VMInstanceDao vmInstanceDao; - @Inject - private PrimaryDataStoreDao storagePool; @Override public boolean isEligible(final Host host) { @@ -117,7 +103,7 @@ public boolean recover(Host r) throws HARecoveryException { public boolean fence(Host r) throws HAFenceException { try { //INVALIDATE CACHE Test - HashMap> poolVolMap = getVolumeUuidOnHost(r); + HashMap> poolVolMap = hostActivityChecker.getVolumeUuidOnHost(r); for (StoragePool pool : poolVolMap.keySet()) { List volume_list = poolVolMap.get(pool); LOG.warn("=====================KVMHAProvider.java===="); @@ -144,31 +130,6 @@ public boolean fence(Host r) throws HAFenceException { } } - private HashMap> getVolumeUuidOnHost(Host r) { - List vm_list = vmInstanceDao.listByHostId(r.getId()); - List volume_list = new ArrayList(); - for (VirtualMachine vm : vm_list) { - LOG.debug(String.format("Retrieving volumes of VM [%s]...", vm.getId())); - List vm_volume_list = volumeDao.findByInstance(vm.getId()); - volume_list.addAll(vm_volume_list); - } - - HashMap> poolVolMap = new HashMap>(); - for (Volume vol : volume_list) { - LOG.debug(String.format("Retrieving storage pool [%s] of volume [%s]...", vol.getPoolId(), vol.getId())); - StoragePool sp = storagePool.findById(vol.getPoolId()); - if (!poolVolMap.containsKey(sp)) { - List list = new ArrayList(); - list.add(vol); - - poolVolMap.put(sp, list); - } else { - poolVolMap.get(sp).add(vol); - } - } - return poolVolMap; - } - @Override public HAResource.ResourceSubType resourceSubType() { return HAResource.ResourceSubType.KVM; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java index e5752cb97da2..1232dd8ef152 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java @@ -188,7 +188,7 @@ protected boolean verifyActivityOfStorageOnHost(HashMap> getVolumeUuidOnHost(Host agent) { + public HashMap> getVolumeUuidOnHost(Host agent) { List vm_list = vmInstanceDao.listByHostId(agent.getId()); List volume_list = new ArrayList(); for (VirtualMachine vm : vm_list) { From 243ca1f7edae97d84810962b0e0c78cd0d467856 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 31 May 2022 13:53:41 +0900 Subject: [PATCH 29/38] =?UTF-8?q?=EC=86=8C=EC=8A=A4=EC=9B=90=EB=B3=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apache/cloudstack/kvm/ha/KVMHAProvider.java | 14 -------------- .../cloudstack/kvm/ha/KVMHostActivityChecker.java | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index fb278a4f2d3f..cb92b053c96b 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -21,8 +21,6 @@ import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.StoragePool; -import com.cloud.storage.Volume; import org.apache.cloudstack.api.response.OutOfBandManagementResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -43,8 +41,6 @@ import javax.inject.Inject; import java.security.InvalidParameterException; -import java.util.HashMap; -import java.util.List; public final class KVMHAProvider extends HAAbstractHostProvider implements HAProvider, Configurable { private final static Logger LOG = Logger.getLogger(KVMHAProvider.class); @@ -102,16 +98,6 @@ public boolean recover(Host r) throws HARecoveryException { @Override public boolean fence(Host r) throws HAFenceException { try { - //INVALIDATE CACHE Test - HashMap> poolVolMap = hostActivityChecker.getVolumeUuidOnHost(r); - for (StoragePool pool : poolVolMap.keySet()) { - List volume_list = poolVolMap.get(pool); - LOG.warn("=====================KVMHAProvider.java===="); - LOG.warn("pool = "+pool); - LOG.warn("volume_list = "+volume_list); - LOG.warn("========================="); - } - if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); if (oobm.getPowerState() == PowerState.Unknown){ diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java index 1232dd8ef152..e5752cb97da2 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java @@ -188,7 +188,7 @@ protected boolean verifyActivityOfStorageOnHost(HashMap> getVolumeUuidOnHost(Host agent) { + private HashMap> getVolumeUuidOnHost(Host agent) { List vm_list = vmInstanceDao.listByHostId(agent.getId()); List volume_list = new ArrayList(); for (VirtualMachine vm : vm_list) { From 6cbc40b8659c7a44431341f07fb4466eb8409c1e Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Thu, 2 Jun 2022 14:38:58 +0900 Subject: [PATCH 30/38] rbd image-cache invalidate --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index baf38f846c60..0d9c5d427b49 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2872,6 +2872,12 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { */ disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); + + // rbd image-cache invalidate + if (data instanceof VolumeObjectTO) { + final VolumeObjectTO volumeObjectTO = (VolumeObjectTO)data; + Script.runSimpleBashScript("rbd image-cache invalidate " + volumeObjectTO.getUuid()); + } } else if (pool.getType() == StoragePoolType.PowerFlex) { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData); } else if (pool.getType() == StoragePoolType.Gluster) { From f484c56183ab324d9785a8fb4f77ae2f8a70b3c8 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Thu, 2 Jun 2022 15:38:02 +0900 Subject: [PATCH 31/38] =?UTF-8?q?volume=20uuid=20->=20path=EB=A1=9C=20?= =?UTF-8?q?=EB=B3=80=EA=B2=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0d9c5d427b49..7a0c6a95c250 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2874,10 +2874,7 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); // rbd image-cache invalidate - if (data instanceof VolumeObjectTO) { - final VolumeObjectTO volumeObjectTO = (VolumeObjectTO)data; - Script.runSimpleBashScript("rbd image-cache invalidate " + volumeObjectTO.getUuid()); - } + Script.runSimpleBashScript("rbd image-cache invalidate " + data.getPath()); } else if (pool.getType() == StoragePoolType.PowerFlex) { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData); } else if (pool.getType() == StoragePoolType.Gluster) { From 2609f2a527e788b82ef0a73709977b1df34473c3 Mon Sep 17 00:00:00 2001 From: dhslove Date: Fri, 17 Jun 2022 14:20:36 +0900 Subject: [PATCH 32/38] Revert "Merge pull request #182 from sjyu1/rbd-primary-ha" This reverts commit 685cfbef52f566fa1a8ce0f2cdfa7b4d781f9fd8, reversing changes made to f2c7ccb0add391837f8d4ee77351426a613016a1. --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 7a0c6a95c250..baf38f846c60 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2872,9 +2872,6 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { */ disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); - - // rbd image-cache invalidate - Script.runSimpleBashScript("rbd image-cache invalidate " + data.getPath()); } else if (pool.getType() == StoragePoolType.PowerFlex) { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData); } else if (pool.getType() == StoragePoolType.Gluster) { From c97c3f97321d4b754bd71a26e1eb5b9e274399c0 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Wed, 23 Nov 2022 13:42:55 +0900 Subject: [PATCH 33/38] Fix conflicting sources --- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 34 ++++++------------- 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 46fb457c2f5d..5f76b34ff294 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -138,16 +138,13 @@ public DataStore initialize(Map dsInfos) { PrimaryDataStoreParameters parameters = new PrimaryDataStoreParameters(); - URI uri = null; - boolean multi = false; + UriUtils.UriInfo uriInfo = UriUtils.getUriInfo(url); + + String scheme = uriInfo.getScheme(); + String storageHost = uriInfo.getStorageHost(); + String storagePath = uriInfo.getStoragePath(); try { - String urlType = url.substring(0, 3); - if (urlType.equals("rbd") && url.contains(",")) { - multi = true; - url = url.replaceAll(",", "/"); - } - uri = new URI(UriUtils.encodeURIComponent(url)); - if (uri.getScheme() == null) { + if (scheme == null) { throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); } else if (uri.getScheme().equalsIgnoreCase("nfs")) { String uriHost = uri.getHost(); @@ -167,23 +164,12 @@ public DataStore initialize(Map dsInfos) { if (uriPath == null) { throw new InvalidParameterValueException("host or path is null, should be sharedmountpoint://localhost/path"); } - } else if (uri.getScheme().equalsIgnoreCase("rbd")) { - String uriHost = uri.getHost(); - String uriPath = uri.getPath(); - if (uriPath == null) { + } else if (scheme.equalsIgnoreCase("rbd")) { + if (storagePath == null) { throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool"); } - if (multi) { - String multiHost = uriHost + (uriPath.substring(0, uriPath.lastIndexOf("/")).replaceAll("/", ",")); - String[] hostArr = multiHost.split(","); - if (hostArr.length > 5) { - throw new InvalidParameterValueException("RADOS monitor can support up to 5 hosts."); - } - } - } else if (uri.getScheme().equalsIgnoreCase("gluster")) { - String uriHost = uri.getHost(); - String uriPath = uri.getPath(); - if (uriHost == null || uriPath == null || uriHost.trim().isEmpty() || uriPath.trim().isEmpty()) { + } else if (scheme.equalsIgnoreCase("gluster")) { + if (storageHost == null || storagePath == null || storageHost.trim().isEmpty() || storagePath.trim().isEmpty()) { throw new InvalidParameterValueException("host or path is null, should be gluster://hostname/volume"); } } From d9a86490604b86fa0a94570a284a6fad4d1b8010 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Wed, 30 Nov 2022 13:45:12 +0900 Subject: [PATCH 34/38] =?UTF-8?q?=ED=8C=8C=EC=9D=B4=EC=8D=AC=20=EB=B2=84?= =?UTF-8?q?=EC=A0=84=20=EC=84=A4=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py | 1 + scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py index 7644f582deef..6ce7d8fe48d1 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py index 33846c19a592..40bdc9a80bb0 100755 --- a/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py +++ b/scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information From f094dff1f041d4410694d0d9f9b3d619296ddff5 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Fri, 16 Dec 2022 13:18:12 +0900 Subject: [PATCH 35/38] =?UTF-8?q?multi=20monitor=20=EC=86=8C=EC=8A=A4?= =?UTF-8?q?=EC=82=AD=EC=A0=9C=20=EB=B0=8F=20=EA=B4=80=EB=A0=A8=ED=8C=8C?= =?UTF-8?q?=EC=9D=BC=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engine/schema/pom.xml | 26 ++++- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 35 +++--- .../main/java/com/cloud/utils/UriUtils.java | 102 ++++++++++++++++-- .../utils/security/ParserUtils.java | 72 +++++++++++++ 4 files changed, 205 insertions(+), 30 deletions(-) create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/security/ParserUtils.java diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index edd664eb4c15..2c51c29cd36b 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -146,7 +146,7 @@ bash templateConfig.sh - ${project.version} + ${project.version} @@ -186,6 +186,30 @@ ${kvm.checksum} + + + + + + + download-vmware-systemvm-template + + + systemvm-vmware + + + + + + org.apache.maven.plugins + maven-resources-plugin + ${cs.resources-plugin.version} + + + com.googlecode.maven-download-plugin + download-maven-plugin + 1.6.3 + download-vmware-template diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 5f76b34ff294..a5c09622e3dc 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -146,22 +146,19 @@ public DataStore initialize(Map dsInfos) { try { if (scheme == null) { throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); - } else if (uri.getScheme().equalsIgnoreCase("nfs")) { - String uriHost = uri.getHost(); - String uriPath = uri.getPath(); - if (uriHost == null || uriPath == null || uriHost.trim().isEmpty() || uriPath.trim().isEmpty()) { + } else if (scheme.equalsIgnoreCase("nfs")) { + if (storageHost == null || storagePath == null || storageHost.trim().isEmpty() || storagePath.trim().isEmpty()) { throw new InvalidParameterValueException("host or path is null, should be nfs://hostname/path"); } - } else if (uri.getScheme().equalsIgnoreCase("cifs")) { + } else if (scheme.equalsIgnoreCase("cifs")) { // Don't validate against a URI encoded URI. URI cifsUri = new URI(url); String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); if (warnMsg != null) { throw new InvalidParameterValueException(warnMsg); } - } else if (uri.getScheme().equalsIgnoreCase("sharedMountPoint")) { - String uriPath = uri.getPath(); - if (uriPath == null) { + } else if (scheme.equalsIgnoreCase("sharedMountPoint")) { + if (storagePath == null) { throw new InvalidParameterValueException("host or path is null, should be sharedmountpoint://localhost/path"); } } else if (scheme.equalsIgnoreCase("rbd")) { @@ -183,24 +180,22 @@ public DataStore initialize(Map dsInfos) { parameters.setTags(tags); parameters.setDetails(details); - String scheme = uri.getScheme(); - String storageHost = uri.getHost(); String hostPath = null; try { - hostPath = URLDecoder.decode(uri.getPath(), "UTF-8"); + hostPath = URLDecoder.decode(storagePath, "UTF-8"); } catch (UnsupportedEncodingException e) { s_logger.error("[ignored] we are on a platform not supporting \"UTF-8\"!?!", e); } if (hostPath == null) { // if decoding fails, use getPath() anyway - hostPath = uri.getPath(); + hostPath = storagePath; } Object localStorage = dsInfos.get("localStorage"); if (localStorage != null) { hostPath = hostPath.replaceFirst("/", ""); hostPath = hostPath.replace("+", " "); } - String userInfo = uri.getUserInfo(); - int port = uri.getPort(); + String userInfo = uriInfo.getUserInfo(); + int port = uriInfo.getPort(); if (s_logger.isDebugEnabled()) { s_logger.debug("createPool Params @ scheme - " + scheme + " storageHost - " + storageHost + " hostPath - " + hostPath + " port - " + port); } @@ -243,10 +238,6 @@ public DataStore initialize(Map dsInfos) { if (port == -1) { port = 0; } - if (multi) { - storageHost = storageHost + (hostPath.substring(0, hostPath.lastIndexOf("/")).replaceAll("/", ",")); - hostPath = hostPath.substring(hostPath.lastIndexOf("/") + 1); - } parameters.setType(StoragePoolType.RBD); parameters.setHost(storageHost); parameters.setPort(port); @@ -321,8 +312,8 @@ public DataStore initialize(Map dsInfos) { parameters.setPort(0); parameters.setPath(hostPath); } else { - s_logger.warn("Unable to figure out the scheme for URI: " + uri); - throw new IllegalArgumentException("Unable to figure out the scheme for URI: " + uri); + s_logger.warn("Unable to figure out the scheme for URI: " + uriInfo); + throw new IllegalArgumentException("Unable to figure out the scheme for URI: " + uriInfo); } } @@ -330,7 +321,7 @@ public DataStore initialize(Map dsInfos) { List pools = primaryDataStoreDao.listPoolByHostPath(storageHost, hostPath); if (!pools.isEmpty() && !scheme.equalsIgnoreCase("sharedmountpoint")) { Long oldPodId = pools.get(0).getPodId(); - throw new CloudRuntimeException("Storage pool " + uri + " already in use by another pod (id=" + oldPodId + ")"); + throw new CloudRuntimeException("Storage pool " + uriInfo + " already in use by another pod (id=" + oldPodId + ")"); } } @@ -584,4 +575,4 @@ public void enableStoragePool(DataStore dataStore) { public void disableStoragePool(DataStore dataStore) { dataStoreHelper.disable(dataStore); } -} +} \ No newline at end of file diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 6b222289b6a6..9a4e78ddaf3c 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -43,6 +43,7 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; @@ -53,6 +54,7 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; import org.apache.commons.httpclient.util.URIUtil; +import org.apache.commons.lang3.StringUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.utils.URLEncodedUtils; @@ -64,12 +66,10 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; - import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; public class UriUtils { @@ -263,6 +263,10 @@ public static Pair validateUrl(String url) throws IllegalArgume } public static Pair validateUrl(String format, String url) throws IllegalArgumentException { + return validateUrl(format, url, false); + } + + public static Pair validateUrl(String format, String url, boolean skipIpv6Check) throws IllegalArgumentException { try { URI uri = new URI(url); if ((uri.getScheme() == null) || @@ -283,7 +287,7 @@ public static Pair validateUrl(String format, String url) throw if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) { throw new IllegalArgumentException("Illegal host specified in url"); } - if (hostAddr instanceof Inet6Address) { + if (!skipIpv6Check && hostAddr instanceof Inet6Address) { throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); } } catch (UnknownHostException uhe) { @@ -372,7 +376,7 @@ public static List getMetalinkChecksums(String url) { protected static Map> getMultipleValuesFromXML(InputStream is, String[] tagNames) { Map> returnValues = new HashMap>(); try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilderFactory factory = ParserUtils.getSaferDocumentBuilderFactory(); DocumentBuilder docBuilder = factory.newDocumentBuilder(); Document doc = docBuilder.parse(is); Element rootElement = doc.getDocumentElement(); @@ -635,4 +639,88 @@ public static List expandPvlanUri(String pvlanRange) { expandedVlans.add(Integer.parseInt(parts[1])); return expandedVlans; } -} + + public static class UriInfo { + String scheme; + String storageHost; + String storagePath; + String userInfo; + int port = -1; + + public UriInfo() { + } + + public UriInfo(String scheme, String storageHost, String storagePath, String userInfo, int port) { + this.scheme = scheme; + this.storageHost = storageHost; + this.storagePath = storagePath; + this.userInfo = userInfo; + this.port = port; + } + + public String getScheme() { + return scheme; + } + + public String getStorageHost() { + return storageHost; + } + + public String getStoragePath() { + return storagePath; + } + + public String getUserInfo() { + return userInfo; + } + + public int getPort() { + return port; + } + + @Override + public String toString() { + return String.format("%s://%s%s%s%s", scheme, + userInfo == null ? "" : userInfo + "@", + storageHost, + port == -1 ? "" : ":" + port, + storagePath == null ? "" : storagePath); + } + } + + public static UriInfo getUriInfo(String url) { + try { + if (url == null) { + return new UriInfo(); + } + if (url.startsWith("rbd://") || url.startsWith("gluefs://")) { + return getRbdUrlInfo(url); + } + URI uri = new URI(UriUtils.encodeURIComponent(url)); + return new UriInfo(uri.getScheme(), uri.getHost(), uri.getPath(), uri.getUserInfo(), uri.getPort()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(url + " is not a valid uri"); + } + } + + private static UriInfo getRbdUrlInfo(String url) { + int secondSlash = StringUtils.ordinalIndexOf(url, "/", 2); + int thirdSlash = StringUtils.ordinalIndexOf(url, "/", 3); + int firstAt = StringUtils.indexOf(url, "@"); + int lastColon = StringUtils.lastIndexOf(url,":"); + int lastSquareBracket = StringUtils.lastIndexOf(url,"]"); + int startOfHost = Math.max(secondSlash, firstAt) + 1; + int endOfHost = lastColon < startOfHost ? (thirdSlash > 0 ? thirdSlash : url.length() + 1) : + (lastSquareBracket > lastColon ? lastSquareBracket + 1 : lastColon); + String storageHosts = StringUtils.substring(url, startOfHost, endOfHost); + String firstHost = storageHosts.split(",")[0]; + String strBeforeHosts = StringUtils.substring(url, 0, startOfHost); + String strAfterHosts = StringUtils.substring(url, endOfHost); + try { + URI uri = new URI(UriUtils.encodeURIComponent(strBeforeHosts + firstHost + strAfterHosts)); + return new UriInfo(uri.getScheme(), storageHosts, uri.getPath(), uri.getUserInfo(), uri.getPort()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(url + " is not a valid uri for RBD"); + } + } +} \ No newline at end of file diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/ParserUtils.java b/utils/src/main/java/org/apache/cloudstack/utils/security/ParserUtils.java new file mode 100644 index 000000000000..dd6a4ea3b252 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/ParserUtils.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.security; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerFactory; + +import org.apache.log4j.Logger; +import org.xml.sax.SAXNotRecognizedException; +import org.xml.sax.SAXNotSupportedException; + +public class ParserUtils { + private static final Logger LOGGER = Logger.getLogger(ParserUtils.class); + + public static DocumentBuilderFactory getSaferDocumentBuilderFactory() throws ParserConfigurationException { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + // REDHAT https://www.blackhat.com/docs/us-15/materials/us-15-Wang-FileCry-The-New-Age-Of-XXE-java-wp.pdf + // OWASP https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html + // and these as well, per Timothy Morgan's 2014 paper: "XML Schema, DTD, and Entity Attacks" + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setXIncludeAware(false); + factory.setExpandEntityReferences(false); + + return factory; + } + + public static TransformerFactory getSaferTransformerFactory() { + TransformerFactory factory = TransformerFactory.newInstance(); + try { + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + } catch (final TransformerConfigurationException e) { + LOGGER.warn("Failed to set safer feature on TransformerFactory: ", e); + } + return factory; + } + + public static SAXParserFactory getSaferSAXParserFactory() { + SAXParserFactory factory = SAXParserFactory.newInstance(); + try { + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + } catch (final ParserConfigurationException | SAXNotRecognizedException | SAXNotSupportedException e) { + LOGGER.warn("Failed to set safer feature on SAXParserFactory: ", e); + } + return factory; + } +} \ No newline at end of file From bbabc991a93284e01a538ef95d77ab424dd7e557 Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Fri, 16 Dec 2022 13:22:19 +0900 Subject: [PATCH 36/38] =?UTF-8?q?poolUUID=EB=A5=BC=20poolUuid=EB=A1=9C=20?= =?UTF-8?q?=EC=B9=B4=EB=A9=9C=EC=BC=80=EC=9D=B4=EC=8A=A4=20=EC=A0=81?= =?UTF-8?q?=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cloud/hypervisor/kvm/resource/KVMHABase.java | 14 +++++++------- .../hypervisor/kvm/resource/KVMHAMonitor.java | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index cdcbf60f61b4..992d7774c56a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -43,14 +43,14 @@ public static enum PoolType { } public static class NfsStoragePool { - String _poolUUID; + String _poolUuid; String _poolIp; String _poolMountSourcePath; String _mountDestPath; PoolType _type; - public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, String mountDestPath, PoolType type) { - _poolUUID = poolUUID; + public NfsStoragePool(String poolUuid, String poolIp, String poolSourcePath, String mountDestPath, PoolType type) { + _poolUuid = poolUuid; _poolIp = poolIp; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; @@ -59,7 +59,7 @@ public NfsStoragePool(String poolUUID, String poolIp, String poolSourcePath, Str } public static class RbdStoragePool { - String _poolUUID; + String _poolUuid; String _monHost; String _poolMountSourcePath; String _mountDestPath; @@ -68,8 +68,8 @@ public static class RbdStoragePool { String _poolAuthSecret; String _poolSourceHost; - public RbdStoragePool(String poolUUID, String monHost, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthUserName, String poolAuthSecret, String poolSourceHost) { - _poolUUID = poolUUID; + public RbdStoragePool(String poolUuid, String monHost, String poolSourcePath, String mountDestPath, PoolType type, String poolAuthUserName, String poolAuthSecret, String poolSourceHost) { + _poolUuid = poolUuid; _monHost = monHost; _poolMountSourcePath = poolSourcePath; _mountDestPath = mountDestPath; @@ -128,7 +128,7 @@ protected String getMountPoint(NfsStoragePool storagePool) { StoragePool pool = null; String poolName = null; try { - pool = LibvirtConnection.getConnection().storagePoolLookupByUUIDString(storagePool._poolUUID); + pool = LibvirtConnection.getConnection().storagePoolLookupByUUIDString(storagePool._poolUuid); if (pool != null) { StoragePoolInfo spi = pool.getInfo(); if (spi.state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 53c9f03c3c06..3a1941d916ba 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -48,9 +48,9 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { public KVMHAMonitor(NfsStoragePool pool, RbdStoragePool rbdpool, String host, String scriptPath, String scriptPathRbd) { if (pool != null) { - nfsstoragePool.put(pool._poolUUID, pool); + nfsstoragePool.put(pool._poolUuid, pool); }else if (rbdpool != null) { - rbdstoragePool.put(rbdpool._poolUUID, rbdpool); + rbdstoragePool.put(rbdpool._poolUuid, rbdpool); } configureHeartBeatPath(scriptPath, scriptPathRbd); hostPrivateIp = host; @@ -64,11 +64,11 @@ private static synchronized void configureHeartBeatPath(String scriptPath, Strin } public void addStoragePool(NfsStoragePool pool) { - nfsstoragePool.put(pool._poolUUID, pool); + nfsstoragePool.put(pool._poolUuid, pool); } public void addStoragePool(RbdStoragePool pool) { - rbdstoragePool.put(pool._poolUUID, pool); + rbdstoragePool.put(pool._poolUuid, pool); } public void removeStoragePool(String uuid) { From f64ad6b9c5c7193068241e51dee2c7b31f35e4da Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Fri, 16 Dec 2022 13:26:19 +0900 Subject: [PATCH 37/38] =?UTF-8?q?if=EB=AC=B8=20space=20=EC=A0=81=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../apache/cloudstack/kvm/ha/KVMHAProvider.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index cb92b053c96b..9b6f1ac529aa 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -54,7 +54,7 @@ public final class KVMHAProvider extends HAAbstractHostProvider implements HAPro @Override public boolean isEligible(final Host host) { - if (outOfBandManagementService.isOutOfBandManagementEnabled(host)){ + if (outOfBandManagementService.isOutOfBandManagementEnabled(host)) { return !isInMaintenanceMode(host) && !isDisabled(host) && hostActivityChecker.getNeighbors(host).length > 0 && (Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType()) || @@ -76,12 +76,12 @@ public boolean hasActivity(final Host r, final DateTime suspectTime) throws HACh @Override public boolean recover(Host r) throws HARecoveryException { try { - if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ + if (outOfBandManagementService.isOutOfBandManagementEnabled(r)) { final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); - if(oobm.getPowerState() == PowerState.Off){ + if (oobm.getPowerState() == PowerState.Off) { LOG.warn("OOBM recover operation failed for the host " + r.getName() + " already OFF"); return false; - }else{ + } else { final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); return resp.getSuccess(); } @@ -89,7 +89,7 @@ public boolean recover(Host r) throws HARecoveryException { LOG.warn("OOBM recover operation failed for the host " + r.getName()); return false; } - } catch (Exception e){ + } catch (Exception e) { LOG.warn("OOBM service is not configured or enabled for this host " + r.getName() + " error is " + e.getMessage()); throw new HARecoveryException(" OOBM service is not configured or enabled for this host " + r.getName(), e); } @@ -98,9 +98,9 @@ public boolean recover(Host r) throws HARecoveryException { @Override public boolean fence(Host r) throws HAFenceException { try { - if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ + if (outOfBandManagementService.isOutOfBandManagementEnabled(r)) { final OutOfBandManagement oobm = outOfBandManagementDao.findByHost(r.getId()); - if (oobm.getPowerState() == PowerState.Unknown){ + if (oobm.getPowerState() == PowerState.Unknown) { return true; } else { final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null); @@ -110,7 +110,7 @@ public boolean fence(Host r) throws HAFenceException { LOG.warn("OOBM fence operation failed for this host " + r.getName()); return false; } - } catch (Exception e){ + } catch (Exception e) { LOG.warn("OOBM service is not configured or enabled for this host " + r.getName() + " error is " + e.getMessage()); throw new HAFenceException("OOBM service is not configured or enabled for this host " + r.getName() , e); } From 53236588f673093528e7d5bd29a34456c40eb12d Mon Sep 17 00:00:00 2001 From: sjyu1 Date: Tue, 27 Dec 2022 09:56:50 +0900 Subject: [PATCH 38/38] =?UTF-8?q?supportHa=20()=20=EB=A9=94=EC=86=8C?= =?UTF-8?q?=EB=93=9C=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kvm/src/main/java/com/cloud/ha/KVMInvestigator.java | 8 ++++++-- ...LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java | 8 +++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java index 816744b067c5..61d74457c0fd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java @@ -80,7 +80,7 @@ public Status isAgentAlive(Host agent) { List clusterPools = _storagePoolDao.listPoolsByCluster(agent.getClusterId()); boolean hasHaPool = false; for (StoragePoolVO pool : clusterPools) { - if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.RBD) { + if (supportsHa(pool.getPoolType())) { hasHaPool = true; break; } @@ -88,7 +88,7 @@ public Status isAgentAlive(Host agent) { if (!hasHaPool) { List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType()); for (StoragePoolVO pool : zonePools) { - if (pool.getPoolType() == StoragePoolType.NetworkFilesystem || pool.getPoolType() == StoragePoolType.RBD) { + if (supportsHa(pool.getPoolType())) { hasHaPool = true; break; } @@ -145,4 +145,8 @@ public Status isAgentAlive(Host agent) { s_logger.debug("HA: HOST is ineligible legacy state " + hostStatus + " for host " + agent.getId()); return hostStatus; } + + public boolean supportsHa(StoragePoolType poolType) { + return poolType.equals(StoragePoolType.NetworkFilesystem) || poolType.equals(StoragePoolType.RBD); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index f0a89d83524c..a01b7202f801 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -27,6 +27,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.ha.KVMInvestigator; import com.cloud.hypervisor.kvm.resource.KVMHABase.NfsStoragePool; import com.cloud.hypervisor.kvm.resource.KVMHABase.RbdStoragePool; import com.cloud.hypervisor.kvm.resource.KVMHAMonitor; @@ -36,15 +37,20 @@ import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage; +import javax.inject.Inject; + @ResourceWrapper(handles = CheckVMActivityOnStoragePoolCommand.class) public final class LibvirtCheckVMActivityOnStoragePoolCommandWrapper extends CommandWrapper { + @Inject + private KVMInvestigator kvmInvestigator; + @Override public Answer execute(final CheckVMActivityOnStoragePoolCommand command, final LibvirtComputingResource libvirtComputingResource) { final ExecutorService executors = Executors.newSingleThreadExecutor(); final KVMHAMonitor monitor = libvirtComputingResource.getMonitor(); final StorageFilerTO pool = command.getPool(); - if (Storage.StoragePoolType.NetworkFilesystem == pool.getType() || Storage.StoragePoolType.RBD == pool.getType()){ + if (kvmInvestigator.supportsHa(pool.getType())) { final NfsStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); final RbdStoragePool rbdpool = monitor.getRbdStoragePool(pool.getUuid()); String vmActivityCheckPath = "";