From 0f7517fdcfe8fa443f83f6f2c79ae277c8cdf59d Mon Sep 17 00:00:00 2001 From: Ben Nemec Date: Thu, 10 Sep 2020 15:01:31 -0500 Subject: [PATCH] Reverse haproxy and keepalived check timings Since we moved to keepalived healthchecking against haproxy, we want haproxy to handle most failures so the VIP doesn't have to move. However, previously the time it took for haproxy to recognize an outage on a node was longer than it was for keepalived, which resulted in the VIP moving before haproxy removed the failing backend. This change makes the haproxy interval 1 second, so it should notice outages in 2 seconds or less (because it has a fall value of 2). The keepalived interval is changed to 2, which means it will detect failures in 2 to 4 seconds (also a fall value of 2). This means haproxy should deal with api outages before keepalived does and allow the VIP to stay on the same node. --- .../baremetal/files/baremetal-haproxy-haproxy.yaml | 2 +- .../baremetal/files/baremetal-keepalived-keepalived.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml b/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml index 015fd626ec..0dc35d4b0d 100644 --- a/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-haproxy-haproxy.yaml @@ -35,5 +35,5 @@ contents: option log-health-checks balance roundrobin {{`{{- range .LBConfig.Backends }} - server {{ .Host }} {{ .Address }}:{{ .Port }} weight 1 verify none check check-ssl inter 3s fall 2 rise 3 + server {{ .Host }} {{ .Address }}:{{ .Port }} weight 1 verify none check check-ssl inter 1s fall 2 rise 3 {{- end }}`}} diff --git a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml index a21fe54b9f..fd14e3b76d 100644 --- a/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml +++ b/templates/master/00-master/baremetal/files/baremetal-keepalived-keepalived.yaml @@ -17,16 +17,16 @@ contents: # api to take the VIP. This isn't preferred because it means all api # traffic will go through one node, but at least it keeps the api available. vrrp_script chk_ocp_lb { - script "/usr/bin/timeout 0.9 /etc/keepalived/chk_ocp_script.sh" - interval 1 + script "/usr/bin/timeout 1.9 /etc/keepalived/chk_ocp_script.sh" + interval 2 weight 20 rise 3 fall 2 } vrrp_script chk_ocp_both { - script "/usr/bin/timeout 0.9 /etc/keepalived/chk_ocp_script_both.sh" - interval 1 + script "/usr/bin/timeout 1.9 /etc/keepalived/chk_ocp_script_both.sh" + interval 2 # Use a smaller weight for this check so it won't trigger the move from # bootstrap to master by itself. weight 5