From 522f4ebac2bc14e17cdf4a62dd2af47a8457361c Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Tue, 10 Nov 2020 20:32:33 +0000
Subject: [PATCH 1/3] vpc: fix ips on wrong interfaces after rebooting vpc vrs

---
 ...VpcVirtualNetworkApplianceManagerImpl.java | 14 ++++++--
 .../debian/opt/cloud/bin/setup/bootstrap.sh   |  3 ++
 ...st_multiple_subnets_in_isolated_network.py | 34 +++++++++++++++++++
 ...ultiple_subnets_in_isolated_network_rvr.py | 34 +++++++++++++++++++
 .../component/test_multiple_subnets_in_vpc.py | 33 ++++++++++++++++++
 .../test_multiple_subnets_in_vpc_rvr.py       | 33 ++++++++++++++++++
 6 files changed, 149 insertions(+), 2 deletions(-)

diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
index e9d32a2152b1..995eb7af451f 100644
--- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
@@ -339,6 +339,7 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine
                     vlanMacAddress.put(vlanTag, routerNic.getMacAddress());
                 }
             }
+            int deviceId = 1; //Public and Guest networks start from device_id = 1
 
             final List<Command> usageCmds = new ArrayList<Command>();
 
@@ -347,7 +348,8 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine
                 // add VPC router to public networks
                 final List<PublicIp> sourceNat = new ArrayList<PublicIp>(1);
                 for (final Pair<Nic, Network> nicNtwk : publicNics) {
-                    final Nic publicNic = nicNtwk.first();
+                    final Nic publicNic = getNicWithUpdatedDeviceId(nicNtwk.first().getId(), deviceId);
+                    deviceId ++;
                     final Network publicNtwk = nicNtwk.second();
                     final IPAddressVO userIp = _ipAddressDao.findByIpAndSourceNetworkId(publicNtwk.getId(), publicNic.getIPv4Address());
 
@@ -385,7 +387,8 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine
 
                 // add VPC router to guest networks
                 for (final Pair<Nic, Network> nicNtwk : guestNics) {
-                    final Nic guestNic = nicNtwk.first();
+                    final Nic guestNic = getNicWithUpdatedDeviceId(nicNtwk.first().getId(), deviceId);
+                    deviceId ++;
                     // plug guest nic
                     final PlugNicCommand plugNicCmd = new PlugNicCommand(_nwHelper.getNicTO(domainRouterVO, guestNic.getNetworkId(), null), domainRouterVO.getInstanceName(), domainRouterVO.getType(), details);
                     cmds.addCommand(plugNicCmd);
@@ -834,4 +837,11 @@ public boolean postStateTransitionEvent(final StateMachine2.Transition<State, Vi
         // once from VpcVirtualNetworkApplianceManagerImpl and once from VirtualNetworkApplianceManagerImpl itself
         return true;
     }
+
+    private Nic getNicWithUpdatedDeviceId(final long nicId, int deviceId) {
+        NicVO nic = _nicDao.findById(nicId);
+        nic.setDeviceId(deviceId);
+        _nicDao.update(nic.getId(), nic);
+        return nic;
+    }
 }
diff --git a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
index 0fb317b32035..769078ea8f09 100755
--- a/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
+++ b/systemvm/debian/opt/cloud/bin/setup/bootstrap.sh
@@ -104,6 +104,9 @@ config_guest() {
           for i in {1..60}; do
             if [ -s $CMDLINE ]; then
               log_it "Received a new non-empty cmdline file from qemu-guest-agent"
+              # Remove old configuration files in /etc/cloudstack if VR is booted from cloudstack
+              rm -rf /etc/cloudstack/*.json
+              log_it "Booting from cloudstack, remove old configuration files in /etc/cloudstack/"
               break
             fi
             sleep 1
diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network.py b/test/integration/component/test_multiple_subnets_in_isolated_network.py
index 0e5054732eeb..9892a3bc6e97 100644
--- a/test/integration/component/test_multiple_subnets_in_isolated_network.py
+++ b/test/integration/component/test_multiple_subnets_in_isolated_network.py
@@ -20,6 +20,7 @@
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ def tearDown(cls):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -672,6 +690,22 @@ def test_01_acquire_public_ips_in_isolated_network_with_single_vr(self):
             self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)
+
         # 20. restart network with cleanup
         self.network1.restart(self.apiclient, cleanup=True)
         routers = self.get_routers(self.network1.id)
diff --git a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py
index b3dd864738a7..2abd076be012 100644
--- a/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py
+++ b/test/integration/component/test_multiple_subnets_in_isolated_network_rvr.py
@@ -20,6 +20,7 @@
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ def tearDown(cls):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -672,6 +690,22 @@ def test_02_acquire_public_ips_in_isolated_network_with_redundant_vrs(self):
             self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth4", False)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth4", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,")
+            guestIp, controlIp, sourcenatIp = self.get_router_ips(router)
+            self.verify_ip_address_in_router(router, host, guestIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth2", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_4.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_5.ipaddress.ipaddress, "eth3", False)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth3", True)
+
         # 20. restart network with cleanup
         self.network1.restart(self.apiclient, cleanup=True)
         routers = self.get_routers(self.network1.id)
diff --git a/test/integration/component/test_multiple_subnets_in_vpc.py b/test/integration/component/test_multiple_subnets_in_vpc.py
index 09f6ec64f892..a9e36966b72e 100644
--- a/test/integration/component/test_multiple_subnets_in_vpc.py
+++ b/test/integration/component/test_multiple_subnets_in_vpc.py
@@ -20,6 +20,7 @@
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ def tearDown(cls):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -790,6 +808,21 @@ def test_03_acquire_public_ips_in_vpc_with_single_vr(self):
             self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
         # 23. restart VPC with cleanup
         self.vpc1.restart(self.apiclient, cleanup=True)
         routers = self.get_vpc_routers(self.vpc1.id)
diff --git a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py
index a03a91b2057d..392620f88cba 100644
--- a/test/integration/component/test_multiple_subnets_in_vpc_rvr.py
+++ b/test/integration/component/test_multiple_subnets_in_vpc_rvr.py
@@ -20,6 +20,7 @@
 """
 
 from nose.plugins.attrib import attr
+from marvin.cloudstackAPI import rebootRouter
 from marvin.cloudstackTestCase import cloudstackTestCase, unittest
 from marvin.lib.utils import (validateList,
                               get_host_credentials,
@@ -106,6 +107,23 @@ def tearDown(cls):
             raise Exception("Warning: Exception during cleanup : %s" % e)
         return
 
+    def get_router(self, router_id):
+        routers = list_routers(
+            self.apiclient,
+            id=router_id,
+            listall=True)
+        self.assertEqual(
+            isinstance(routers, list),
+            True,
+            "Check for list routers response return valid data"
+        )
+        self.assertNotEqual(
+            len(routers),
+            0,
+            "Check list router response"
+        )
+        return routers[0]
+
     def get_routers(self, network_id):
         routers = list_routers(
             self.apiclient,
@@ -790,6 +808,21 @@ def test_04_acquire_public_ips_in_vpc_with_redundant_vrs(self):
             self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
             self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth5", True)
 
+        # reboot router
+        for router in routers:
+            cmd = rebootRouter.rebootRouterCmd()
+            cmd.id = router.id
+            self.apiclient.rebootRouter(cmd)
+            router = self.get_router(router.id)
+            host = self.get_router_host(router)
+            self.verify_network_interfaces_in_router(router, host, "eth0,eth1,eth2,eth3,eth4,")
+            controlIp, sourcenatIp, tier1_Ip, tier2_Ip = self.get_vpc_router_ips(router)
+            self.verify_ip_address_in_router(router, host, controlIp, "eth0", True)
+            self.verify_ip_address_in_router(router, host, sourcenatIp, "eth1", True)
+            self.verify_ip_address_in_router(router, host, ipaddress_6.ipaddress.ipaddress, "eth2", True)
+            self.verify_ip_address_in_router(router, host, tier1_Ip, "eth3", True)
+            self.verify_ip_address_in_router(router, host, tier2_Ip, "eth4", True)
+
         # 23. restart VPC with cleanup
         self.vpc1.restart(self.apiclient, cleanup=True)
         routers = self.get_vpc_routers(self.vpc1.id)

From eba9c867b8ff6473d7eec9aa7fa997fe0576f743 Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Fri, 13 Nov 2020 12:42:52 +0000
Subject: [PATCH 2/3]  #4467: Rename to updateNicWithDeviceId

---
 .../router/VpcVirtualNetworkApplianceManagerImpl.java       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
index 995eb7af451f..556fe18388f1 100644
--- a/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
+++ b/server/src/main/java/com/cloud/network/router/VpcVirtualNetworkApplianceManagerImpl.java
@@ -348,7 +348,7 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine
                 // add VPC router to public networks
                 final List<PublicIp> sourceNat = new ArrayList<PublicIp>(1);
                 for (final Pair<Nic, Network> nicNtwk : publicNics) {
-                    final Nic publicNic = getNicWithUpdatedDeviceId(nicNtwk.first().getId(), deviceId);
+                    final Nic publicNic = updateNicWithDeviceId(nicNtwk.first().getId(), deviceId);
                     deviceId ++;
                     final Network publicNtwk = nicNtwk.second();
                     final IPAddressVO userIp = _ipAddressDao.findByIpAndSourceNetworkId(publicNtwk.getId(), publicNic.getIPv4Address());
@@ -387,7 +387,7 @@ public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachine
 
                 // add VPC router to guest networks
                 for (final Pair<Nic, Network> nicNtwk : guestNics) {
-                    final Nic guestNic = getNicWithUpdatedDeviceId(nicNtwk.first().getId(), deviceId);
+                    final Nic guestNic = updateNicWithDeviceId(nicNtwk.first().getId(), deviceId);
                     deviceId ++;
                     // plug guest nic
                     final PlugNicCommand plugNicCmd = new PlugNicCommand(_nwHelper.getNicTO(domainRouterVO, guestNic.getNetworkId(), null), domainRouterVO.getInstanceName(), domainRouterVO.getType(), details);
@@ -838,7 +838,7 @@ public boolean postStateTransitionEvent(final StateMachine2.Transition<State, Vi
         return true;
     }
 
-    private Nic getNicWithUpdatedDeviceId(final long nicId, int deviceId) {
+    private Nic updateNicWithDeviceId(final long nicId, int deviceId) {
         NicVO nic = _nicDao.findById(nicId);
         nic.setDeviceId(deviceId);
         _nicDao.update(nic.getId(), nic);

From f6038166c5c1a46fe383a9ee059ae397d948baff Mon Sep 17 00:00:00 2001
From: Wei Zhou <w.zhou@global.leaseweb.com>
Date: Tue, 17 Nov 2020 20:26:46 +0000
Subject: [PATCH 3/3] CLSTACK-8923 vr: Force a restart of keepalived if
 conntrackd is not running or configuration has changed

---
 systemvm/debian/opt/cloud/bin/cs/CsRedundant.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py
index cefedbfe84f1..190de1ab82e2 100755
--- a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py
+++ b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py
@@ -173,7 +173,7 @@ def _redundant_on(self):
         force_keepalived_restart = False
         proc = CsProcess(['/etc/conntrackd/conntrackd.conf'])
 
-        if not proc.find() and not is_equals:
+        if not proc.find() or not is_equals:
             CsHelper.copy(conntrackd_template_conf, self.CONNTRACKD_CONF)
             CsHelper.service("conntrackd", "restart")
             force_keepalived_restart = True