From c6578a58229ede7b9952dea0e942fa182ee706e6 Mon Sep 17 00:00:00 2001 From: "Brad P. Crochet" Date: Mon, 27 Jan 2020 08:44:20 -0500 Subject: [PATCH] Add support to unicast keepalived The current deployment workflow relies on the network allowing multicast traffic for VRRP management of the VIP addresses, this PR updates keepalived to support also unicast mode. To enable unicast keepalived, keepalived.conf in each node should include all the relevant nodes (including the bootstrap node ) IP addresses, to support this requirement the bootstrap node will run a simple server that will return its IP address, rest of nodes details will be retrieved from kube-apiserver. The keepalived mode (unicast or multicast) controlled by 'ENABLE_UNICAST' environment variable, it's set by default to multicast. --- manifests/baremetal/keepalived.conf.tmpl | 24 ++- manifests/baremetal/keepalived.yaml | 125 +++++++++++---- pkg/operator/assets/bindata.go | 149 +++++++++++++----- .../baremetal/files/baremetal-keepalived.yaml | 28 +++- .../baremetal-keepalived-keepalived.yaml | 19 +++ .../baremetal-keepalived-keepalived.yaml | 10 ++ 6 files changed, 277 insertions(+), 78 deletions(-) diff --git a/manifests/baremetal/keepalived.conf.tmpl b/manifests/baremetal/keepalived.conf.tmpl index 5d2ec43b42..115143cec9 100644 --- a/manifests/baremetal/keepalived.conf.tmpl +++ b/manifests/baremetal/keepalived.conf.tmpl @@ -4,17 +4,29 @@ # For more information, see installer/data/data/bootstrap/baremetal/README.md # in the installer repo. -vrrp_instance {{`{{.Cluster.Name}}`}}_API { +{{`{{$nonVirtualIP := .NonVirtualIP}}`}} + +{{`vrrp_instance {{.Cluster.Name}}_API { state BACKUP - interface {{`{{.VRRPInterface}}`}} - virtual_router_id {{`{{.Cluster.APIVirtualRouterID }}`}} + interface {{.VRRPInterface}} + virtual_router_id {{.Cluster.APIVirtualRouterID }} priority 50 advert_int 1 + {{ if .EnableUnicast }} + unicast_src_ip {{.NonVirtualIP}} + unicast_peer { + {{range .LBConfig.Backends}} + {{if ne $nonVirtualIP .Address}}{{.Address}}{{end}} + {{else}} + {{.NonVirtualIP}} + {{end}} + } + {{end}} authentication { auth_type PASS - auth_pass {{`{{.Cluster.Name}}`}}_api_vip + auth_pass {{.Cluster.Name}}_api_vip } virtual_ipaddress { - {{`{{ .Cluster.APIVIP }}`}}/{{`{{ .Cluster.VIPNetmask }}`}} + {{ .Cluster.APIVIP }}/{{ .Cluster.VIPNetmask }} } -} +}`}} diff --git a/manifests/baremetal/keepalived.yaml b/manifests/baremetal/keepalived.yaml index cbc92ba8d7..14804b3539 100644 --- a/manifests/baremetal/keepalived.yaml +++ b/manifests/baremetal/keepalived.yaml @@ -21,50 +21,56 @@ spec: - name: manifests hostPath: path: "/opt/openshift/manifests" - initContainers: - - name: render-config + - name: run-dir + empty-dir: {} + containers: + - name: keepalived-unicast image: {{ .Images.BaremetalRuntimeCfgBootstrap }} command: - - runtimecfg - - render - - "/etc/kubernetes/kubeconfig" + - unicastipserver - "--api-vip" - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }}" - "--ingress-vip" - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.IngressIP }}" - - "/config" - - "--out-dir" - - "/etc/keepalived" - - "--cluster-config" - - "/opt/openshift/manifests/cluster-config.yaml" - resources: {} - volumeMounts: - - name: resource-dir - mountPath: "/config" - - name: kubeconfig - mountPath: "/etc/kubernetes/kubeconfig" - - name: conf-dir - mountPath: "/etc/keepalived" - - name: manifests - mountPath: "/opt/openshift/manifests" - imagePullPolicy: IfNotPresent - containers: - name: keepalived securityContext: privileged: true - image: {{ .Images.KeepalivedBootstrap }} + image: {{.Images.KeepalivedBootstrap}} env: - name: NSS_SDB_USE_CACHE value: "no" command: - - /usr/sbin/keepalived - args: - - "-f" - - "/etc/keepalived/keepalived.conf" - - "--dont-fork" - - "--vrrp" - - "--log-detail" - - "--log-console" + - /bin/bash + - -c + - | + #/bin/bash + reload_keepalived() + { + if pid=$(pgrep -o keepalived); then + kill -s SIGHUP "$pid" + else + /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & + fi + } + msg_handler() + { + while read -r line; do + echo "The client sent: $line" >&2 + # currently only 'reload' msg is supported + if [ "$line" = reload ]; then + reload_keepalived + fi + done + } + set -ex + declare -r keepalived_sock="/var/run/keepalived/keepalived.sock" + export -f msg_handler + export -f reload_keepalived + if [ -s "/etc/keepalived/keepalived.conf" ]; then + /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & + fi + rm -f "$keepalived_sock" + socat UNIX-LISTEN:${keepalived_sock},fork system:'bash -c msg_handler' resources: requests: cpu: 100m @@ -72,8 +78,65 @@ spec: volumeMounts: - name: conf-dir mountPath: "/etc/keepalived" + - name: run-dir + mountPath: "/var/run/keepalived" + livenessProbe: + exec: + command: + - /bin/bash + - -c + - | + [[ -s /etc/keepalived/keepalived.conf ]] || \ + kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data + initialDelaySeconds: 20 terminationMessagePolicy: FallbackToLogsOnError imagePullPolicy: IfNotPresent + - name: keepalived-monitor + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + env: + - name: ENABLE_UNICAST + value: "no" + - name: IS_BOOTSTRAP + value: "yes" + command: + - dynkeepalived + - "/etc/kubernetes/kubeconfig" + - "/config/keepalived.conf.tmpl" + - "/etc/keepalived/keepalived.conf" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.IngressIP }}" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + - "--check-interval" + - "5s" + resources: + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + - name: run-dir + mountPath: "/var/run/keepalived" + - name: manifests + mountPath: "/opt/openshift/manifests" + readinessProbe: + httpGet: + path: /readyz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + imagePullPolicy: IfNotPresent hostNetwork: true tolerations: - operator: Exists diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 76ed9a442b..18d63812f5 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -260,20 +260,32 @@ var _manifestsBaremetalKeepalivedConfTmpl = []byte(`# Configuration template for # For more information, see installer/data/data/bootstrap/baremetal/README.md # in the installer repo. -vrrp_instance {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_API { +{{`+"`"+`{{$nonVirtualIP := .NonVirtualIP}}`+"`"+`}} + +{{`+"`"+`vrrp_instance {{.Cluster.Name}}_API { state BACKUP - interface {{`+"`"+`{{.VRRPInterface}}`+"`"+`}} - virtual_router_id {{`+"`"+`{{.Cluster.APIVirtualRouterID }}`+"`"+`}} + interface {{.VRRPInterface}} + virtual_router_id {{.Cluster.APIVirtualRouterID }} priority 50 advert_int 1 + {{ if .EnableUnicast }} + unicast_src_ip {{.NonVirtualIP}} + unicast_peer { + {{range .LBConfig.Backends}} + {{if ne $nonVirtualIP .Address}}{{.Address}}{{end}} + {{else}} + {{.NonVirtualIP}} + {{end}} + } + {{end}} authentication { auth_type PASS - auth_pass {{`+"`"+`{{.Cluster.Name}}`+"`"+`}}_api_vip + auth_pass {{.Cluster.Name}}_api_vip } virtual_ipaddress { - {{`+"`"+`{{ .Cluster.APIVIP }}`+"`"+`}}/{{`+"`"+`{{ .Cluster.VIPNetmask }}`+"`"+`}} + {{ .Cluster.APIVIP }}/{{ .Cluster.VIPNetmask }} } -} +}`+"`"+`}} `) func manifestsBaremetalKeepalivedConfTmplBytes() ([]byte, error) { @@ -314,50 +326,56 @@ spec: - name: manifests hostPath: path: "/opt/openshift/manifests" - initContainers: - - name: render-config + - name: run-dir + empty-dir: {} + containers: + - name: keepalived-unicast image: {{ .Images.BaremetalRuntimeCfgBootstrap }} command: - - runtimecfg - - render - - "/etc/kubernetes/kubeconfig" + - unicastipserver - "--api-vip" - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }}" - "--ingress-vip" - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.IngressIP }}" - - "/config" - - "--out-dir" - - "/etc/keepalived" - - "--cluster-config" - - "/opt/openshift/manifests/cluster-config.yaml" - resources: {} - volumeMounts: - - name: resource-dir - mountPath: "/config" - - name: kubeconfig - mountPath: "/etc/kubernetes/kubeconfig" - - name: conf-dir - mountPath: "/etc/keepalived" - - name: manifests - mountPath: "/opt/openshift/manifests" - imagePullPolicy: IfNotPresent - containers: - name: keepalived securityContext: privileged: true - image: {{ .Images.KeepalivedBootstrap }} + image: {{.Images.KeepalivedBootstrap}} env: - name: NSS_SDB_USE_CACHE value: "no" command: - - /usr/sbin/keepalived - args: - - "-f" - - "/etc/keepalived/keepalived.conf" - - "--dont-fork" - - "--vrrp" - - "--log-detail" - - "--log-console" + - /bin/bash + - -c + - | + #/bin/bash + reload_keepalived() + { + if pid=$(pgrep -o keepalived); then + kill -s SIGHUP "$pid" + else + /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & + fi + } + msg_handler() + { + while read -r line; do + echo "The client sent: $line" >&2 + # currently only 'reload' msg is supported + if [ "$line" = reload ]; then + reload_keepalived + fi + done + } + set -ex + declare -r keepalived_sock="/var/run/keepalived/keepalived.sock" + export -f msg_handler + export -f reload_keepalived + if [ -s "/etc/keepalived/keepalived.conf" ]; then + /usr/sbin/keepalived -f /etc/keepalived/keepalived.conf --dont-fork --vrrp --log-detail --log-console & + fi + rm -f "$keepalived_sock" + socat UNIX-LISTEN:${keepalived_sock},fork system:'bash -c msg_handler' resources: requests: cpu: 100m @@ -365,8 +383,65 @@ spec: volumeMounts: - name: conf-dir mountPath: "/etc/keepalived" + - name: run-dir + mountPath: "/var/run/keepalived" + livenessProbe: + exec: + command: + - /bin/bash + - -c + - | + [[ -s /etc/keepalived/keepalived.conf ]] || \ + kill -s SIGUSR1 "$(pgrep -o keepalived)" && ! grep -q "State = FAULT" /tmp/keepalived.data + initialDelaySeconds: 20 terminationMessagePolicy: FallbackToLogsOnError imagePullPolicy: IfNotPresent + - name: keepalived-monitor + image: {{ .Images.BaremetalRuntimeCfgBootstrap }} + env: + - name: ENABLE_UNICAST + value: "no" + - name: IS_BOOTSTRAP + value: "yes" + command: + - dynkeepalived + - "/etc/kubernetes/kubeconfig" + - "/config/keepalived.conf.tmpl" + - "/etc/keepalived/keepalived.conf" + - "--api-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.APIServerInternalIP }}" + - "--ingress-vip" + - "{{ .ControllerConfig.Infra.Status.PlatformStatus.BareMetal.IngressIP }}" + - "--cluster-config" + - "/opt/openshift/manifests/cluster-config.yaml" + - "--check-interval" + - "5s" + resources: + requests: + cpu: 100m + memory: 200Mi + volumeMounts: + - name: resource-dir + mountPath: "/config" + - name: kubeconfig + mountPath: "/etc/kubernetes/kubeconfig" + - name: conf-dir + mountPath: "/etc/keepalived" + - name: run-dir + mountPath: "/var/run/keepalived" + - name: manifests + mountPath: "/opt/openshift/manifests" + readinessProbe: + httpGet: + path: /readyz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 + imagePullPolicy: IfNotPresent hostNetwork: true tolerations: - operator: Exists diff --git a/templates/common/baremetal/files/baremetal-keepalived.yaml b/templates/common/baremetal/files/baremetal-keepalived.yaml index 43e717a750..8d9cda18de 100644 --- a/templates/common/baremetal/files/baremetal-keepalived.yaml +++ b/templates/common/baremetal/files/baremetal-keepalived.yaml @@ -18,7 +18,10 @@ contents: path: "/etc/kubernetes/static-pod-resources/keepalived" - name: kubeconfig hostPath: - path: "/etc/kubernetes/kubeconfig" + path: "/etc/kubernetes" + - name: kubeconfigvarlib + hostPath: + path: "/var/lib/kubelet" - name: conf-dir hostPath: path: "/etc/keepalived" @@ -87,10 +90,17 @@ contents: terminationMessagePolicy: FallbackToLogsOnError imagePullPolicy: IfNotPresent - name: keepalived-monitor + securityContext: + privileged: true image: {{ .Images.baremetalRuntimeCfgImage }} + env: + - name: ENABLE_UNICAST + value: "no" + - name: IS_BOOTSTRAP + value: "no" command: - dynkeepalived - - "/etc/kubernetes/kubeconfig" + - "/var/lib/kubelet/kubeconfig" - "/config/keepalived.conf.tmpl" - "/etc/keepalived/keepalived.conf" - "--api-vip" @@ -104,12 +114,22 @@ contents: volumeMounts: - name: resource-dir mountPath: "/config" - - name: kubeconfig - mountPath: "/etc/kubernetes/kubeconfig" + - name: kubeconfigvarlib + mountPath: "/var/lib/kubelet" - name: conf-dir mountPath: "/etc/keepalived" - name: run-dir mountPath: "/var/run/keepalived" + readinessProbe: + httpGet: + path: /readyz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 10 imagePullPolicy: IfNotPresent hostNetwork: true tolerations: diff --git a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml index e879638f98..ca7726052e 100644 --- a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml @@ -21,12 +21,23 @@ contents: weight 50 } + {{`{{$nonVirtualIP := .NonVirtualIP}}`}} + vrrp_instance {{`{{ .Cluster.Name }}`}}_API { state BACKUP interface {{`{{ .VRRPInterface }}`}} virtual_router_id {{`{{ .Cluster.APIVirtualRouterID }}`}} priority 40 advert_int 1 + {{`{{if .EnableUnicast}}`}} + unicast_src_ip {{`{{.NonVirtualIP}}`}} + unicast_peer { + {{`{{ .BootstrapIP }}`}} + {{`{{range .LBConfig.Backends}} + {{if ne $nonVirtualIP .Address}}{{.Address}}{{end}} + {{end}}`}} + } + {{`{{end}}`}} authentication { auth_type PASS auth_pass {{`{{ .Cluster.Name }}`}}_api_vip @@ -45,6 +56,14 @@ contents: virtual_router_id {{`{{ .Cluster.IngressVirtualRouterID }}`}} priority 40 advert_int 1 + {{`{{if .EnableUnicast}}`}} + unicast_src_ip {{`{{.NonVirtualIP}}`}} + unicast_peer { + {{`{{range .IngressConfig.Peers}} + {{if ne $nonVirtualIP .}}{{.}}{{end}} + {{end}}`}} + } + {{`{{end}}`}} authentication { auth_type PASS auth_pass {{`{{ .Cluster.Name }}`}}_ingress_vip diff --git a/templates/worker/00-worker/baremetal/files/baremetal-keepalived-keepalived.yaml b/templates/worker/00-worker/baremetal/files/baremetal-keepalived-keepalived.yaml index 82b9072477..1f21c30347 100644 --- a/templates/worker/00-worker/baremetal/files/baremetal-keepalived-keepalived.yaml +++ b/templates/worker/00-worker/baremetal/files/baremetal-keepalived-keepalived.yaml @@ -10,12 +10,22 @@ contents: weight 50 } + {{`{{$nonVirtualIP := .NonVirtualIP}}`}} + vrrp_instance {{`{{ .Cluster.Name }}`}}_INGRESS { state BACKUP interface {{`{{ .VRRPInterface }}`}} virtual_router_id {{`{{ .Cluster.IngressVirtualRouterID }}`}} priority 40 advert_int 1 + {{`{{if .EnableUnicast}}`}} + unicast_src_ip {{`{{.NonVirtualIP}}`}} + unicast_peer { + {{`{{range .IngressConfig.Peers}} + {{if ne $nonVirtualIP .}}{{.}}{{end}} + {{end}}`}} + } + {{`{{end}}`}} authentication { auth_type PASS auth_pass {{`{{ .Cluster.Name }}`}}_ingress_vip