diff --git a/manager/state/raft/raft.go b/manager/state/raft/raft.go index 3e3410fe04..18ffcac2bb 100644 --- a/manager/state/raft/raft.go +++ b/manager/state/raft/raft.go @@ -6,6 +6,8 @@ import ( "math" "math/rand" "net" + "os" + "strconv" "sync" "sync/atomic" "time" @@ -40,6 +42,14 @@ import ( ) var ( + // EnvRaftHeartbeatTick is the env to set to customize the Raft heartbeat tick + EnvRaftHeartbeatTick = "SWARMKIT_X_RAFT_HEARTBEAT_TICK" + + // EnvRaftElectionTick is the env to set to customize the Raft leader election delay + // when a follower does not hear from the leader for this many ticks. Typically + // set it to be 10x the HeartbeatTick + EnvRaftElectionTick = "SWARMKIT_X_RAFT_ELECTION_TICK" + // ErrNoRaftMember is thrown when the node is not yet part of a raft cluster ErrNoRaftMember = errors.New("raft: node is not yet part of a raft cluster") // ErrConfChangeRefused is returned when there is an issue with the configuration change @@ -470,15 +480,30 @@ func (n *Node) joinCluster(ctx context.Context) error { return nil } +func intFromEnv(key string, defaultValue int) int { + env := os.Getenv(key) + if env == "" { + return defaultValue + } + + value, err := strconv.Atoi(env) + if err != nil { + // This is for initialization. If the env is set incorrectly panic and exit + // the process now. + panic(err) + } + return value +} + // DefaultNodeConfig returns the default config for a // raft node that can be modified and customized func DefaultNodeConfig() *raft.Config { return &raft.Config{ - HeartbeatTick: 1, + HeartbeatTick: intFromEnv(EnvRaftHeartbeatTick, 1), // Recommended value in etcd/raft is 10 x (HeartbeatTick). // Lower values were seen to have caused instability because of // frequent leader elections when running on flakey networks. - ElectionTick: 10, + ElectionTick: intFromEnv(EnvRaftElectionTick, 10), MaxSizePerMsg: math.MaxUint16, MaxInflightMsgs: 256, Logger: log.L, @@ -495,8 +520,8 @@ func DefaultRaftConfig() api.RaftConfig { // Recommended value in etcd/raft is 10 x (HeartbeatTick). // Lower values were seen to have caused instability because of // frequent leader elections when running on flakey networks. - HeartbeatTick: 1, - ElectionTick: 10, + HeartbeatTick: uint32(intFromEnv(EnvRaftHeartbeatTick, 1)), + ElectionTick: uint32(intFromEnv(EnvRaftElectionTick, 10)), } } diff --git a/manager/state/raft/raft_test.go b/manager/state/raft/raft_test.go index b33baa91a7..27408896b0 100644 --- a/manager/state/raft/raft_test.go +++ b/manager/state/raft/raft_test.go @@ -62,6 +62,30 @@ func TestMain(m *testing.M) { os.Exit(res) } +func TestDefaultConfigsFromEnv(t *testing.T) { + os.Setenv(raft.EnvRaftHeartbeatTick, "10") + os.Setenv(raft.EnvRaftElectionTick, "100") + + config := raft.DefaultNodeConfig() + assert.Equal(t, config.HeartbeatTick, 10) + assert.Equal(t, config.ElectionTick, 100) + + rconfig := raft.DefaultRaftConfig() + assert.Equal(t, rconfig.HeartbeatTick, uint32(10)) + assert.Equal(t, rconfig.ElectionTick, uint32(100)) + + os.Unsetenv(raft.EnvRaftHeartbeatTick) + os.Unsetenv(raft.EnvRaftElectionTick) + + config = raft.DefaultNodeConfig() + assert.Equal(t, config.HeartbeatTick, 1) + assert.Equal(t, config.ElectionTick, 10) + + rconfig = raft.DefaultRaftConfig() + assert.Equal(t, rconfig.HeartbeatTick, uint32(1)) + assert.Equal(t, rconfig.ElectionTick, uint32(10)) +} + func TestRaftBootstrap(t *testing.T) { t.Parallel()