diff --git a/cmd/activator/main.go b/cmd/activator/main.go index be4a39dcfcdc..cb09a35a8b21 100644 --- a/cmd/activator/main.go +++ b/cmd/activator/main.go @@ -43,9 +43,11 @@ import ( const ( maxUploadBytes = 32e6 // 32MB - same as app engine - maxRetries = 60 - retryInterval = 1 * time.Second logLevelKey = "activator" + + maxRetries = 18 // the sum of all retries would add up to 1 minute + minRetryInterval = 100 * time.Millisecond + exponentialBackoffBase = 1.3 ) func main() { @@ -96,7 +98,7 @@ func main() { // a small delay for k8s to include the ready IP in service. // https://github.com/knative/serving/issues/660#issuecomment-384062553 shouldRetry := activatorutil.RetryStatus(http.StatusServiceUnavailable) - retryer := activatorutil.NewLinearRetryer(retryInterval, maxRetries) + retryer := activatorutil.NewRetryer(activatorutil.NewExponentialIntervalFunc(minRetryInterval, exponentialBackoffBase), maxRetries) rt := activatorutil.NewRetryRoundTripper(activatorutil.AutoTransport, logger, retryer, shouldRetry) diff --git a/pkg/activator/util/retryer.go b/pkg/activator/util/retryer.go index 7357ed8ea6bd..195214aac3e7 100644 --- a/pkg/activator/util/retryer.go +++ b/pkg/activator/util/retryer.go @@ -13,26 +13,60 @@ limitations under the License. package util -import "time" +import ( + "math" + "time" +) -type Retryer interface { - Retry(func() bool) int -} +// RetryerFunc is a function that wraps an action to be +// retried. +type RetryerFunc func(ActionFunc) int +// ActionFunc is a function that is retried by a `Retryer`. +// Returns true iff succeeded, false if not. type ActionFunc func() bool -type RetryerFunc func(ActionFunc) int -func (r RetryerFunc) Retry(f func() bool) int { +// IntervalFunc is a function that calculates an interval +// given the number of retries that already happened. +type IntervalFunc func(int) time.Duration + +// Retryer is an entity that can retry a given `ActionFunc`. +type Retryer interface { + Retry(ActionFunc) int +} + +// Retry invokes 1 retry on `f` +func (r RetryerFunc) Retry(f ActionFunc) int { return r(f) } -// NewLinearRetryer will return a retryer that retries `action` up to -// `maxRetries` times with `interval` delay between retries -func NewLinearRetryer(interval time.Duration, maxRetries int) Retryer { +// NewRetryer creates a function where `action` will be retried +// at most `maxRetries` times, with an interval calculated in +// between retries by the `intervalFunc` +func NewRetryer(intervalFunc IntervalFunc, maxRetries int) Retryer { return RetryerFunc(func(action ActionFunc) (retries int) { for retries = 1; !action() && retries < maxRetries; retries++ { - time.Sleep(interval) + time.Sleep(intervalFunc(retries)) } return }) } + +// NewLinearIntervalFunc creates a function always returning +// a static value `interval` +func NewLinearIntervalFunc(interval time.Duration) IntervalFunc { + return func(_ int) time.Duration { + return interval + } +} + +// NewExponentialIntervalFunc creates a function returning a +// `time.Duration`, that represents the time to wait in between +// two retries, calculated as `minInterval * (base ^ retries)` +func NewExponentialIntervalFunc(minInterval time.Duration, base float64) IntervalFunc { + return func(retries int) time.Duration { + retryIntervalMs := float64(minInterval / time.Millisecond) + multiplicator := math.Pow(base, float64(retries)) + return time.Duration(int(retryIntervalMs*multiplicator)) * time.Millisecond + } +} diff --git a/pkg/activator/util/retryer_test.go b/pkg/activator/util/retryer_test.go index 5f669097c087..96477625907a 100644 --- a/pkg/activator/util/retryer_test.go +++ b/pkg/activator/util/retryer_test.go @@ -17,17 +17,7 @@ import ( "time" ) -func TestLinearRetry(t *testing.T) { - checkInterval := func(last *time.Time, want time.Duration) { - now := time.Now() - got := now.Sub(*last) - *last = now - - if got < want { - t.Errorf("Unexpected retry interval. Want %v, got %v", want, got) - } - } - +func TestRetryer(t *testing.T) { examples := []struct { label string interval time.Duration @@ -37,28 +27,24 @@ func TestLinearRetry(t *testing.T) { }{ { label: "atleast once", - interval: 5 * time.Millisecond, maxRetries: 0, responses: []bool{true}, wantRetries: 1, }, { label: "< maxRetries", - interval: 5 * time.Millisecond, maxRetries: 3, responses: []bool{false, true}, wantRetries: 2, }, { label: "= maxRetries", - interval: 10 * time.Millisecond, maxRetries: 3, responses: []bool{false, false, true}, wantRetries: 3, }, { label: "> maxRetries", - interval: 5 * time.Millisecond, maxRetries: 3, responses: []bool{false, false, false, true}, wantRetries: 3, @@ -67,19 +53,16 @@ func TestLinearRetry(t *testing.T) { for _, e := range examples { t.Run(e.label, func(t *testing.T) { - var lastRetry time.Time var got int a := func() bool { - checkInterval(&lastRetry, e.interval) - ok := e.responses[got] got++ return ok } - lr := NewLinearRetryer(e.interval, e.maxRetries) + lr := NewRetryer(func(_ int) time.Duration { return 0 }, e.maxRetries) reported := lr.Retry(a) @@ -93,3 +76,74 @@ func TestLinearRetry(t *testing.T) { }) } } + +func TestLinearIntervalFunc(t *testing.T) { + interval := 100 * time.Millisecond + examples := []struct { + label string + retries int + expectedInterval time.Duration + }{ + { + label: "linear: 1 retry", + retries: 1, + expectedInterval: interval, + }, + { + label: "linear: 3 retries", + retries: 3, + expectedInterval: interval, + }, + } + + for _, e := range examples { + t.Run(e.label, func(t *testing.T) { + intervalFunc := NewLinearIntervalFunc(interval) + got := intervalFunc(e.retries) + + if got != e.expectedInterval { + t.Errorf("Unexpected interval. Want %d, got %d", e.expectedInterval, got) + } + }) + } +} + +func TestExponentialIntervalFunc(t *testing.T) { + minInterval := 100 * time.Millisecond + examples := []struct { + label string + base float64 + retries int + expectedInterval time.Duration + }{ + { + label: "exponential: 1 retry", + base: 1.3, + retries: 1, + expectedInterval: 130 * time.Millisecond, + }, + { + label: "exponential: 3 retries", + base: 1.3, + retries: 3, + expectedInterval: 219 * time.Millisecond, + }, + { + label: "exponential: 10 retries", + base: 1.3, + retries: 10, + expectedInterval: 1378 * time.Millisecond, + }, + } + + for _, e := range examples { + t.Run(e.label, func(t *testing.T) { + intervalFunc := NewExponentialIntervalFunc(minInterval, e.base) + got := intervalFunc(e.retries) + + if got != e.expectedInterval { + t.Errorf("Unexpected interval. Want %d, got %d", e.expectedInterval, got) + } + }) + } +}