From 28ee74366b6baa44a1520bf878f4450f70c65344 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Thu, 21 May 2020 14:42:17 -0500 Subject: [PATCH 1/3] Switch haproxy healthcheck from /readyz to /haproxyready It seems that /readyz is intercepting our healthchecks to the api, even though monitor-uri is only defined in a different listen block. This is probably good anyway as it clarifies that this is not a healthcheck against the kubernetes api. --- .../00-master/baremetal/files/baremetal-haproxy-haproxy.yaml | 2 +- .../master/00-master/baremetal/files/baremetal-haproxy.yaml | 2 +- .../baremetal/files/baremetal-keepalived-keepalived.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml b/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml index 950011bfec..2f0e62883d 100644 --- a/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml @@ -20,7 +20,7 @@ contents: listen health_check_http_url bind :::50936 v4v6 mode http - monitor-uri /readyz + monitor-uri /haproxyready option dontlognull listen stats bind localhost:{{`{{ .LBConfig.StatPort }}`}} diff --git a/templates/master/00-master/baremetal/files/baremetal-haproxy.yaml b/templates/master/00-master/baremetal/files/baremetal-haproxy.yaml index 385e03f1a1..020ec2635d 100644 --- a/templates/master/00-master/baremetal/files/baremetal-haproxy.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-haproxy.yaml @@ -100,7 +100,7 @@ contents: livenessProbe: initialDelaySeconds: 10 httpGet: - path: /readyz + path: /haproxyready port: 50936 terminationMessagePolicy: FallbackToLogsOnError imagePullPolicy: IfNotPresent diff --git a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml index 71c7e16499..d71cd24fcd 100644 --- a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml @@ -3,7 +3,7 @@ path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" contents: inline: | vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLfs https://localhost:6443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" + script "/usr/bin/curl -o /dev/null -kLfs https://localhost:6443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/haproxyready" interval 1 weight 50 } From 2795754868dc31df959f17697eb1665bc9902b5e Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Wed, 20 May 2020 14:38:06 -0500 Subject: [PATCH 2/3] Change keepalived healthcheck to loadbalanced API endpoint We don't necessarily want to move the API VIP just because the API server on the local node goes down. HAProxy will happily continue to distribute traffic to other nodes (as long as one is still up) and moving the VIP may cause connection disruptions like the ones we are seeing in CI. This switches the keepalived healthcheck to look at port 9443, which is where HAProxy listens, instead of 6443, which is just the local API server. In order for this to work sanely, it was also necessary to change the timings on the healthchecks in keepalived and HAProxy. Previously, HAProxy had a check interval of 3 seconds and required two failing checks to mark a backend down. This meant up to 6 seconds to notice a dead backend. Keepalived had a check interval of only 1 second and no grace period for failed tests, which means it would trigger a failover much sooner than 6 seconds. Depending on how HAProxy chooses to distribute requests, it's possible the keepalived healthcheck would get sent to the down backend and trigger a VIP failover. Since we'd prefer to let HAProxy handle outages (as moving the VIP may break connections), let's swap those timings so that HAProxy is checking every second with a fall value of 2 (so two failed tests to mark backend down), while keepalived checks every 2 seconds, also with a fall value of 2. This should give HAProxy time to notice an outage before the second keepalived healthcheck (HAProxy will respond within 2 seconds, where keepalived will take between 2 and 4 seconds). This should still result in acceptably quick failover but hopefully avoid doing so unnecessarily. --- .../00-master/baremetal/files/baremetal-haproxy-haproxy.yaml | 2 +- .../baremetal/files/baremetal-keepalived-keepalived.yaml | 4 +++- .../openstack/files/openstack-keepalived-keepalived.yaml | 2 +- .../00-master/ovirt/files/ovirt-keepalived-keepalived.yaml | 2 +- .../vsphere/files/vsphere-keepalived-keepalived.yaml | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml b/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml index 2f0e62883d..b7802bbba2 100644 --- a/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml @@ -35,5 +35,5 @@ contents: option log-health-checks balance roundrobin {{`{{- range .LBConfig.Backends }} - server {{ .Host }} {{ .Address }}:{{ .Port }} weight 1 verify none check check-ssl inter 3s fall 2 rise 3 + server {{ .Host }} {{ .Address }}:{{ .Port }} weight 1 verify none check check-ssl inter 1s fall 2 rise 3 {{- end }}`}} diff --git a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml index d71cd24fcd..a9b4c5c546 100644 --- a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml @@ -3,9 +3,11 @@ path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" contents: inline: | vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLfs https://localhost:6443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/haproxyready" + script "/usr/bin/curl -o /dev/null -kLfs https://localhost:9443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/haproxyready" interval 1 weight 50 + rise 3 + fall 2 } # TODO: Improve this check. The port is assumed to be alive. diff --git a/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml b/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml index 71c7e16499..a044ddbb4b 100644 --- a/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml +++ b/templates/master/00-master/openstack/files/openstack-keepalived-keepalived.yaml @@ -3,7 +3,7 @@ path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" contents: inline: | vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLfs https://localhost:6443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" + script "/usr/bin/curl -o /dev/null -kLfs https://localhost:9443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" interval 1 weight 50 } diff --git a/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml b/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml index 2602159748..6eff7bbeeb 100644 --- a/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml +++ b/templates/master/00-master/ovirt/files/ovirt-keepalived-keepalived.yaml @@ -3,7 +3,7 @@ path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" contents: inline: | vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLfs https://0:6443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" + script "/usr/bin/curl -o /dev/null -kLfs https://0:9443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" interval 1 weight 50 } diff --git a/templates/master/00-master/vsphere/files/vsphere-keepalived-keepalived.yaml b/templates/master/00-master/vsphere/files/vsphere-keepalived-keepalived.yaml index 98e3027cc1..f2ead14ea0 100644 --- a/templates/master/00-master/vsphere/files/vsphere-keepalived-keepalived.yaml +++ b/templates/master/00-master/vsphere/files/vsphere-keepalived-keepalived.yaml @@ -8,7 +8,7 @@ contents: {{ if .Infra.Status.PlatformStatus.VSphere -}} {{ if .Infra.Status.PlatformStatus.VSphere.APIServerInternalIP -}} vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLfs https://localhost:6443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" + script "/usr/bin/curl -o /dev/null -kLfs https://localhost:9443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/readyz" interval 1 weight 50 } From df7426933102eec77158d2785e65a805c9a1f5d0 Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Mon, 8 Jun 2020 14:49:49 -0500 Subject: [PATCH 3/3] Split vrrp_scripts for api and haproxy checks Currently it's difficult/impossible to tell which check caused a failover of the VIP. Split the check so there's a separate one for each service and we can see which failed. The weight for each check is also changed to 6. The reason for this number is that the bootstrap's priority is 50, while the masters start at 40. Each passing vrrp_script adds its weight to the priority, and since we don't want the masters to take the VIP until both haproxy and the api are up, we need to make sure that the master priority is only >50 when both checks pass. --- .../files/baremetal-keepalived-keepalived.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml index a9b4c5c546..5a8e6e2aa0 100644 --- a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml @@ -3,9 +3,17 @@ path: "/etc/kubernetes/static-pod-resources/keepalived/keepalived.conf.tmpl" contents: inline: | vrrp_script chk_ocp { - script "/usr/bin/curl -o /dev/null -kLfs https://localhost:9443/readyz && /usr/bin/curl -o /dev/null -kLfs http://localhost:50936/haproxyready" + script "/usr/bin/curl -o /dev/null -kLfs https://localhost:9443/readyz" interval 1 - weight 50 + weight 6 + rise 3 + fall 2 + } + + vrrp_script chk_haproxy { + script "/usr/bin/curl -o /dev/null -kLfs http://localhost:50936/haproxyready" + interval 1 + weight 6 rise 3 fall 2 } @@ -33,6 +41,7 @@ contents: } track_script { chk_ocp + chk_haproxy } }