Watch firewall reload in sysconfigwatcher

assets/components/openshift-router/deployment.yaml

@@ -63,6 +63,8 @@ spec:
               value: 1s
             - name: ROUTER_DOMAIN
               value: apps.{{ .BaseDomain }}
+            - name: ROUTER_CREATION_TIMESTAMP
+              value: {{ .CreationTimestamp }}
           livenessProbe:
             httpGet:
               path: /healthz

assets/components/ovn/master/daemonset.yaml

@@ -328,7 +328,7 @@ spec:
           fi
 
           # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes
-          echo "$(date -Iseconds) - starting ovnkube-master, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}"
+          echo "$(date -Iseconds) - starting ovnkube-master at {{ .CreationTimestamp }}, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}"
 
           echo "I$(date "+%m%d %H:%M:%S.%N") - copy ovn-k8s-cni-overlay"
           cp -f /usr/libexec/cni/ovn-k8s-cni-overlay /cni-bin-dir/

docs/howto_sysconf_watch.md

@@ -1,6 +1,10 @@
 # MicroShift Mitigation of System Configuration Changes
 
-MicroShift depends on the device IP address and system-wide clock settings to remain consistent during its runtime. However, these settings may occasionally change on edge devices (i.e. DHCP or NTP updates). When such changes occur, some MicroShift components may stop functioning properly. To mitigate this situation, MicroShift monitors the mentioned system configuration settings and restarts if a setting change is detected.
+MicroShift depends on the following system settings to remain consistent during its runtime:
+- Device IP address
+- System-wide clock settings
+- Iptable configurations
+However, these settings may occasionally change on edge devices (i.e. DHCP or NTP updates). When such changes occur, some MicroShift components may stop functioning properly. To mitigate this situation, MicroShift monitors the mentioned system configuration settings, restarts or reloads components if a setting change is detected.
 
 This document describes how to simulate system configuration changes in a virtual environment and verify that MicroShift service reacts by restarting when necessary.
 
@@ -138,4 +142,22 @@ The below (non-proportional!) graph shows when certificates are rotated.
   certificate will be rotated for a new one.
 
 If the rotated certificate is a CA, all of the certificates it signed get rotated
-as well.
+as well.
+
+## Firewall Changes
+
+Reload the firewall rules with the following command to trigger the reloading of MicroShift components.
+
+```bash
+sudo firewall-cmd --reload
+```
+
+Firewall reload action flushes the iptable configurations which results in failed network traffic. <br>
+Run the `journalctl -xu microshift` command to verify that the components are reloaded. The logs should contain reload messages.
+
+```
+Dec 21 08:57:01 localhost.localdomain microshift[2005232]: infrastructure-services-manager I1221 08:57:01.567046 2005232 iptables.go:590] iptables canary mangle/MICROSHIFT-CANARY deleted
+Dec 21 08:57:01 localhost.localdomain microshift[2005232]: infrastructure-services-manager W1221 08:57:01.582233 2005232 components.go:25] Iptables flush is detected, reloading affected components
+Dec 21 08:57:01 localhost.localdomain microshift[2005232]: infrastructure-services-manager I1221 08:57:01.582276 2005232 components.go:64] Reload ingress controller
+Dec 21 08:57:01 localhost.localdomain microshift[2005232]: infrastructure-services-manager I1221 08:57:01.644365 2005232 components.go:69] Reload CNI plugin
+```

pkg/components/components.go

@@ -1,15 +1,37 @@
 package components
 
 import (
+	"time"
+
 	"github.com/openshift/microshift/pkg/config"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
+	"k8s.io/kubernetes/pkg/util/iptables"
 )
 
+const iptablesCheckInterval = time.Second * 5
+
 var microshiftDataDir = config.GetDataDir()
 
-func StartComponents(cfg *config.MicroshiftConfig) error {
+func StartComponents(cfg *config.MicroshiftConfig, iptClients []iptables.Interface) error {
 	kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin)
 
+	for i := range iptClients {
+		iptClient := iptClients[i]
+		go iptClient.Monitor(
+			iptables.Chain("MICROSHIFT-CANARY"),
+			[]iptables.Table{iptables.TableMangle, iptables.TableNAT, iptables.TableFilter},
+			func() {
+				klog.Warningf("Iptables flush is detected, reloading affected components")
+				if err := reloadOnIptableFlush(cfg); err != nil {
+					klog.Errorf("Failed to reload affected components: %v", err)
+				}
+			},
+			iptablesCheckInterval,
+			wait.NeverStop,
+		)
+	}
+
 	if err := startServiceCAController(cfg, kubeAdminConfig); err != nil {
 		klog.Warningf("Failed to start service-ca controller: %v", err)
 		return err
@@ -35,3 +57,19 @@ func StartComponents(cfg *config.MicroshiftConfig) error {
 	}
 	return nil
 }
+
+func reloadOnIptableFlush(cfg *config.MicroshiftConfig) error {
+	kubeAdminConfig := cfg.KubeConfigPath(config.KubeAdmin)
+
+	klog.Infof("Reload ingress controller")
+	if err := startIngressController(cfg, kubeAdminConfig); err != nil {
+		klog.Warningf("Failed to reload ingress router controller: %v", err)
+		return err
+	}
+	klog.Infof("Reload CNI plugin")
+	if err := startCNIPlugin(cfg, kubeAdminConfig); err != nil {
+		klog.Warningf("Failed to reload CNI plugin: %v", err)
+		return err
+	}
+	return nil
+}

pkg/components/controllers.go

@@ -2,6 +2,7 @@ package components
 
 import (
 	"os"
+	"time"
 
 	"github.com/openshift/microshift/pkg/assets"
 	"github.com/openshift/microshift/pkg/config"
@@ -162,7 +163,10 @@ func startIngressController(cfg *config.MicroshiftConfig, kubeconfigPath string)
 		return err
 	}
 
-	if err := assets.ApplyDeployments(apps, renderTemplate, renderParamsFromConfig(cfg, nil), kubeconfigPath); err != nil {
+	extraParams := assets.RenderParams{
+		"CreationTimestamp": time.Now().Format("2006-01-02 15:04:05"),
+	}
+	if err := assets.ApplyDeployments(apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {
 		klog.Warningf("Failed to apply apps %v: %v", apps, err)
 		return err
 	}

pkg/components/networking.go

@@ -3,6 +3,7 @@ package components
 import (
 	"fmt"
 	"path/filepath"
+	"time"
 
 	"github.com/openshift/microshift/pkg/assets"
 	"github.com/openshift/microshift/pkg/config"
@@ -77,9 +78,10 @@ func startCNIPlugin(cfg *config.MicroshiftConfig, kubeconfigPath string) error {
 		return err
 	}
 	extraParams := assets.RenderParams{
-		"OVNConfig":      ovnConfig,
-		"KubeconfigPath": kubeconfigPath,
-		"KubeconfigDir":  filepath.Join(microshiftDataDir, "/resources/kubeadmin"),
+		"OVNConfig":         ovnConfig,
+		"KubeconfigPath":    kubeconfigPath,
+		"KubeconfigDir":     filepath.Join(microshiftDataDir, "/resources/kubeadmin"),
+		"CreationTimestamp": time.Now().Format("2006-01-02 15:04:05"),
 	}
 	if err := assets.ApplyConfigMaps(cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {
 		klog.Warningf("Failed to apply configMap %v %v", cm, err)

pkg/controllers/infra-services-controller.go

@@ -23,15 +23,25 @@ import (
 	"github.com/openshift/microshift/pkg/assets"
 	"github.com/openshift/microshift/pkg/components"
 	"github.com/openshift/microshift/pkg/config"
+	"k8s.io/kubernetes/pkg/util/iptables"
+	"k8s.io/utils/exec"
 )
 
 type InfrastructureServicesManager struct {
-	cfg *config.MicroshiftConfig
+	cfg        *config.MicroshiftConfig
+	iptClients []iptables.Interface
 }
 
 func NewInfrastructureServices(cfg *config.MicroshiftConfig) *InfrastructureServicesManager {
 	s := &InfrastructureServicesManager{}
 	s.cfg = cfg
+
+	// Initialize iptables util
+	exec := exec.New()
+	s.iptClients = []iptables.Interface{
+		iptables.New(exec, iptables.ProtocolIPv4),
+		iptables.New(exec, iptables.ProtocolIPv6),
+	}
 	return s
 }
 
@@ -55,7 +65,7 @@ func (s *InfrastructureServicesManager) Run(ctx context.Context, ready chan<- st
 	}
 
 	// TO-DO add readiness check
-	if err := components.StartComponents(s.cfg); err != nil {
+	if err := components.StartComponents(s.cfg, s.iptClients); err != nil {
 		return err
 	}
 	klog.Infof("%s launched ocp componets", s.Name())

scripts/auto-rebase/rebase.sh

@@ -598,6 +598,8 @@ update_manifests() {
     yq -i '.spec.template.spec.containers[0].env += {"name": "ROUTER_USE_PROXY_PROTOCOL", "value": "false"}' "${REPOROOT}"/assets/components/openshift-router/deployment.yaml
     yq -i '.spec.template.spec.containers[0].env += {"name": "GRACEFUL_SHUTDOWN_DELAY", "value": "1s"}' "${REPOROOT}"/assets/components/openshift-router/deployment.yaml
     yq -i '.spec.template.spec.containers[0].env += {"name": "ROUTER_DOMAIN", "value": "apps.REPLACE_CLUSTER_DOMAIN"}' "${REPOROOT}"/assets/components/openshift-router/deployment.yaml
+    #    Add creation timestamp to trigger force rollout on MicroShift restart
+    yq -i '.spec.template.spec.containers[0].env += {"name": "ROUTER_CREATION_TIMESTAMP", "value": "{{ .CreationTimestamp }}"}' "${REPOROOT}"/assets/components/openshift-router/deployment.yaml
     # 4) Replace MicroShift templating vars (do this last, as yq trips over Go templates)
     sed -i 's|REPLACE_CLUSTER_DOMAIN|{{ .BaseDomain }}|g' "${REPOROOT}"/assets/components/openshift-router/deployment.yaml
     sed -i 's|REPLACE_ROUTER_IMAGE|{{ .ReleaseImage.haproxy_router }}|' "${REPOROOT}"/assets/components/openshift-router/deployment.yaml