Backoff retries in the activator.

cmd/activator/main.go

@@ -43,9 +43,11 @@ import (
 
 const (
 	maxUploadBytes = 32e6 // 32MB - same as app engine
-	maxRetries     = 60
-	retryInterval  = 1 * time.Second
 	logLevelKey    = "activator"
+
+	maxRetries             = 18 // the sum of all retries would add up to 1 minute
+	minRetryInterval       = 100 * time.Millisecond
+	exponentialBackoffBase = 1.3
 )
 
 func main() {
@@ -96,7 +98,7 @@ func main() {
 	// a small delay for k8s to include the ready IP in service.
 	// https://github.com/knative/serving/issues/660#issuecomment-384062553
 	shouldRetry := activatorutil.RetryStatus(http.StatusServiceUnavailable)
-	retryer := activatorutil.NewLinearRetryer(retryInterval, maxRetries)
+	retryer := activatorutil.NewRetryer(activatorutil.NewExponentialIntervalFunc(minRetryInterval, exponentialBackoffBase), maxRetries)
 
 	rt := activatorutil.NewRetryRoundTripper(activatorutil.AutoTransport, logger, retryer, shouldRetry)
 

pkg/activator/util/retryer.go

@@ -13,26 +13,60 @@ limitations under the License.
 
 package util
 
-import "time"
+import (
+	"math"
+	"time"
+)
 
-type Retryer interface {
-	Retry(func() bool) int
-}
+// RetryerFunc is a function that wraps an action to be
+// retried.
+type RetryerFunc func(ActionFunc) int
 
+// ActionFunc is a function that is retried by a `Retryer`.
+// Returns true iff succeeded, false if not.
 type ActionFunc func() bool
-type RetryerFunc func(ActionFunc) int
 
-func (r RetryerFunc) Retry(f func() bool) int {
+// IntervalFunc is a function that calculates an interval
+// given the number of retries that already happened.
+type IntervalFunc func(int) time.Duration
+
+// Retryer is an entity that can retry a given `ActionFunc`.
+type Retryer interface {
+	Retry(ActionFunc) int
+}
+
+// Retry invokes 1 retry on `f`
+func (r RetryerFunc) Retry(f ActionFunc) int {
 	return r(f)
 }
 
-// NewLinearRetryer will return a retryer that retries `action` up to
-// `maxRetries` times with `interval` delay between retries
-func NewLinearRetryer(interval time.Duration, maxRetries int) Retryer {
+// NewRetryer creates a function where `action` will be retried
+// at most `maxRetries` times, with an interval calculated in
+// between retries by the `intervalFunc`
+func NewRetryer(intervalFunc IntervalFunc, maxRetries int) Retryer {
 	return RetryerFunc(func(action ActionFunc) (retries int) {
 		for retries = 1; !action() && retries < maxRetries; retries++ {
-			time.Sleep(interval)
+			time.Sleep(intervalFunc(retries))
 		}
 		return
 	})
 }
+
+// NewLinearIntervalFunc creates a function always returning
+// a static value `interval`
+func NewLinearIntervalFunc(interval time.Duration) IntervalFunc {
+	return func(_ int) time.Duration {
+		return interval
+	}
+}
+
+// NewExponentialIntervalFunc creates a function returning a
+// `time.Duration`, that represents the time to wait in between
+// two retries, calculated as `minInterval * (base ^ retries)`
+func NewExponentialIntervalFunc(minInterval time.Duration, base float64) IntervalFunc {
+	return func(retries int) time.Duration {
+		retryIntervalMs := float64(minInterval / time.Millisecond)
+		multiplicator := math.Pow(base, float64(retries))
+		return time.Duration(int(retryIntervalMs*multiplicator)) * time.Millisecond
+	}
+}

pkg/activator/util/retryer_test.go

@@ -17,17 +17,7 @@ import (
 	"time"
 )
 
-func TestLinearRetry(t *testing.T) {
-	checkInterval := func(last *time.Time, want time.Duration) {
-		now := time.Now()
-		got := now.Sub(*last)
-		*last = now
-
-		if got < want {
-			t.Errorf("Unexpected retry interval. Want %v, got %v", want, got)
-		}
-	}
-
+func TestRetryer(t *testing.T) {
 	examples := []struct {
 		label       string
 		interval    time.Duration
@@ -37,28 +27,24 @@ func TestLinearRetry(t *testing.T) {
 	}{
 		{
 			label:       "atleast once",
-			interval:    5 * time.Millisecond,
 			maxRetries:  0,
 			responses:   []bool{true},
 			wantRetries: 1,
 		},
 		{
 			label:       "< maxRetries",
-			interval:    5 * time.Millisecond,
 			maxRetries:  3,
 			responses:   []bool{false, true},
 			wantRetries: 2,
 		},
 		{
 			label:       "= maxRetries",
-			interval:    10 * time.Millisecond,
 			maxRetries:  3,
 			responses:   []bool{false, false, true},
 			wantRetries: 3,
 		},
 		{
 			label:       "> maxRetries",
-			interval:    5 * time.Millisecond,
 			maxRetries:  3,
 			responses:   []bool{false, false, false, true},
 			wantRetries: 3,
@@ -67,19 +53,16 @@ func TestLinearRetry(t *testing.T) {
 
 	for _, e := range examples {
 		t.Run(e.label, func(t *testing.T) {
-			var lastRetry time.Time
 			var got int
 
 			a := func() bool {
-				checkInterval(&lastRetry, e.interval)
-
 				ok := e.responses[got]
 				got++
 
 				return ok
 			}
 
-			lr := NewLinearRetryer(e.interval, e.maxRetries)
+			lr := NewRetryer(func(_ int) time.Duration { return 0 }, e.maxRetries)
 
 			reported := lr.Retry(a)
 
@@ -93,3 +76,74 @@ func TestLinearRetry(t *testing.T) {
 		})
 	}
 }
+
+func TestLinearIntervalFunc(t *testing.T) {
+	interval := 100 * time.Millisecond
+	examples := []struct {
+		label            string
+		retries          int
+		expectedInterval time.Duration
+	}{
+		{
+			label:            "linear: 1 retry",
+			retries:          1,
+			expectedInterval: interval,
+		},
+		{
+			label:            "linear: 3 retries",
+			retries:          3,
+			expectedInterval: interval,
+		},
+	}
+
+	for _, e := range examples {
+		t.Run(e.label, func(t *testing.T) {
+			intervalFunc := NewLinearIntervalFunc(interval)
+			got := intervalFunc(e.retries)
+
+			if got != e.expectedInterval {
+				t.Errorf("Unexpected interval. Want %d, got %d", e.expectedInterval, got)
+			}
+		})
+	}
+}
+
+func TestExponentialIntervalFunc(t *testing.T) {
+	minInterval := 100 * time.Millisecond
+	examples := []struct {
+		label            string
+		base             float64
+		retries          int
+		expectedInterval time.Duration
+	}{
+		{
+			label:            "exponential: 1 retry",
+			base:             1.3,
+			retries:          1,
+			expectedInterval: 130 * time.Millisecond,
+		},
+		{
+			label:            "exponential: 3 retries",
+			base:             1.3,
+			retries:          3,
+			expectedInterval: 219 * time.Millisecond,
+		},
+		{
+			label:            "exponential: 10 retries",
+			base:             1.3,
+			retries:          10,
+			expectedInterval: 1378 * time.Millisecond,
+		},
+	}
+
+	for _, e := range examples {
+		t.Run(e.label, func(t *testing.T) {
+			intervalFunc := NewExponentialIntervalFunc(minInterval, e.base)
+			got := intervalFunc(e.retries)
+
+			if got != e.expectedInterval {
+				t.Errorf("Unexpected interval. Want %d, got %d", e.expectedInterval, got)
+			}
+		})
+	}
+}