From 96a8c14afe81357e05d53f566c2c5cb219d9d8d4 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 14:07:46 -0600 Subject: [PATCH 01/10] Added upload progress bar tracker for ISO images. Removed concurrent upload since it doesn't make any significant performance imapact. When I tried to measure performance differene with and without concurrent uppload, the results were fluctuating in a wide range so not good measurement was possible. --- lib/install/management/create.go | 147 +++++++++++++--------------- lib/progresslog/progresslog.go | 86 ++++++++++++++++ lib/progresslog/progresslog_test.go | 88 +++++++++++++++++ 3 files changed, 241 insertions(+), 80 deletions(-) create mode 100644 lib/progresslog/progresslog.go create mode 100644 lib/progresslog/progresslog_test.go diff --git a/lib/install/management/create.go b/lib/install/management/create.go index 0a9b7b1099..8d77dacca7 100644 --- a/lib/install/management/create.go +++ b/lib/install/management/create.go @@ -18,10 +18,12 @@ import ( "context" "fmt" "path" - "sync" + "path/filepath" "time" "github.com/vmware/govmomi/object" + "github.com/vmware/vic/lib/progresslog" + "github.com/vmware/vic/lib/config" "github.com/vmware/vic/lib/install/data" "github.com/vmware/vic/lib/install/opsuser" @@ -120,100 +122,85 @@ func (d *Dispatcher) uploadImages(files map[string]string) error { // upload the images d.op.Info("Uploading images for container") - results := make(chan error, len(files)) - var wg sync.WaitGroup - for key, image := range files { - - wg.Add(1) - go func(key string, image string) { - finalMessage := "" - d.op.Infof("\t%q", image) - - // upload function that is passed to retry - operationForRetry := func() error { - // attempt to delete the iso image first in case of failed upload - dc := d.session.Datacenter - fm := d.session.Datastore.NewFileManager(dc, false) - ds := d.session.Datastore - - isoTargetPath := path.Join(d.vmPathName, key) - // check iso first - d.op.Debugf("target delete path = %s", isoTargetPath) - _, err := ds.Stat(d.op, isoTargetPath) - if err != nil { - switch err.(type) { - case object.DatastoreNoSuchFileError: - d.op.Debug("File not found. Nothing to do.") - case object.DatastoreNoSuchDirectoryError: - d.op.Debug("Directory not found. Nothing to do.") - default: - // otherwise force delete - err := fm.Delete(d.op, isoTargetPath) - if err != nil { - d.op.Debugf("Failed to delete image (%s) with error (%s)", image, err.Error()) - return err - } + baseName := filepath.Base(image) + finalMessage := "" + // upload function that is passed to retry + isoTargetPath := path.Join(d.vmPathName, key) + + operationForRetry := func() error { + // attempt to delete the iso image first in case of failed upload + dc := d.session.Datacenter + fm := d.session.Datastore.NewFileManager(dc, false) + ds := d.session.Datastore + + // check iso first + d.op.Debugf("Checking if file already exists: %s", isoTargetPath) + _, err := ds.Stat(d.op, isoTargetPath) + if err != nil { + switch err.(type) { + case object.DatastoreNoSuchFileError: + d.op.Debug("File not found. Nothing to do.") + case object.DatastoreNoSuchDirectoryError: + d.op.Debug("Directory not found. Nothing to do.") + default: + d.op.Debugf("ISO file already exists, deleting: %s", isoTargetPath) + err := fm.Delete(d.op, isoTargetPath) + if err != nil { + d.op.Debugf("Failed to delete image (%s) with error (%s)", image, err.Error()) + return err } } - - return d.session.Datastore.UploadFile(d.op, image, path.Join(d.vmPathName, key), nil) } - // counter for retry decider - retryCount := uploadRetryLimit + d.op.Infof("Uploading %s as %s", baseName, key) - // decider for our retry, will retry the upload uploadRetryLimit times - uploadRetryDecider := func(err error) bool { - if err == nil { - return false - } + ul := progresslog.NewUploadLogger(d.op.Infof, baseName, time.Second*3) + // need to wait since UploadLogger is asynchronous. + defer ul.Wait() - retryCount-- - if retryCount < 0 { - d.op.Warnf("Attempted upload a total of %d times without success, Upload process failed.", uploadRetryLimit) - return false - } - d.op.Warnf("failed an attempt to upload isos with err (%s), %d retries remain", err.Error(), retryCount) - return true - } + return d.session.Datastore.UploadFile(d.op, image, path.Join(d.vmPathName, key), + progresslog.UploadParams(ul)) + } - // Build retry config - backoffConf := retry.NewBackoffConfig() - backoffConf.InitialInterval = uploadInitialInterval - backoffConf.MaxInterval = uploadMaxInterval - backoffConf.MaxElapsedTime = uploadMaxElapsedTime - - uploadErr := retry.DoWithConfig(operationForRetry, uploadRetryDecider, backoffConf) - if uploadErr != nil { - finalMessage = fmt.Sprintf("\t\tUpload failed for %q: %s\n", image, uploadErr) - if d.force { - finalMessage = fmt.Sprintf("%s\t\tContinuing despite failures (due to --force option)\n", finalMessage) - finalMessage = fmt.Sprintf("%s\t\tNote: The VCH will not function without %q...", finalMessage, image) - results <- errors.New(finalMessage) - } else { - results <- errors.New(finalMessage) - } + // counter for retry decider + retryCount := uploadRetryLimit + + // decider for our retry, will retry the upload uploadRetryLimit times + uploadRetryDecider := func(err error) bool { + if err == nil { + return false } - wg.Done() - }(key, image) - } - wg.Wait() - close(results) + retryCount-- + if retryCount < 0 { + d.op.Warnf("Attempted upload a total of %d times without success, Upload process failed.", uploadRetryLimit) + return false + } + d.op.Warnf("failed an attempt to upload isos with err (%s), %d retries remain", err.Error(), retryCount) + return true + } - uploadFailed := false - for err := range results { - if err != nil { - d.op.Error(err) - uploadFailed = true + // Build retry config + backoffConf := retry.NewBackoffConfig() + backoffConf.InitialInterval = uploadInitialInterval + backoffConf.MaxInterval = uploadMaxInterval + backoffConf.MaxElapsedTime = uploadMaxElapsedTime + + uploadErr := retry.DoWithConfig(operationForRetry, uploadRetryDecider, backoffConf) + if uploadErr != nil { + finalMessage = fmt.Sprintf("\t\tUpload failed for %q: %s\n", image, uploadErr) + if d.force { + finalMessage = fmt.Sprintf("%s\t\tContinuing despite failures (due to --force option)\n", finalMessage) + finalMessage = fmt.Sprintf("%s\t\tNote: The VCH will not function without %q...", finalMessage, image) + } + d.op.Error(finalMessage) + return errors.New("Failed to upload iso images successfully.") } - } - if uploadFailed { - return errors.New("Failed to upload iso images successfully.") } return nil + } // cleanupAfterCreationFailed cleans up the dangling resource pool for the failed VCH and any bridge network if there is any. diff --git a/lib/progresslog/progresslog.go b/lib/progresslog/progresslog.go new file mode 100644 index 0000000000..4bb2d9f3e2 --- /dev/null +++ b/lib/progresslog/progresslog.go @@ -0,0 +1,86 @@ +package progresslog + +import ( + "sync" + "time" + + "github.com/vmware/govmomi/vim25/progress" + "github.com/vmware/govmomi/vim25/soap" +) + +// UploadParams uses default upload settings as initial input and set uploadLogger as a logger. +func UploadParams(ul *uploadLogger) *soap.Upload { + params := soap.DefaultUpload + params.Progress = ul + return ¶ms +} + +// uploadLogger io used to track upload progress to ESXi/VC of a specific file. +type uploadLogger struct { + wg sync.WaitGroup + filename string + interval time.Duration + logTo func(format string, args ...interface{}) +} + +// NewUploadLogger returns a new instance of uploadLogger. User can provide a logger interface +// that will be used to dump output of the current upload status. +func NewUploadLogger(logTo func(format string, args ...interface{}), + filename string, progressInterval time.Duration) *uploadLogger { + + return &uploadLogger{ + logTo: logTo, + filename: filename, + interval: progressInterval, + } +} + +// Sink returns a channel that receives current upload progress status. +// Channel has to be closed by the caller. +func (ul *uploadLogger) Sink() chan<- progress.Report { + ul.wg.Add(1) + ch := make(chan progress.Report) + fmtStr := "Uploading %s. Progress: %.2f%%" + + go func() { + var curProgress float32 + var lastProgress float32 + ul.logTo(fmtStr, ul.filename, curProgress) + + mu := sync.Mutex{} + ticker := time.NewTicker(ul.interval) + + // Print progress every 3 + go func() { + for range ticker.C { + mu.Lock() + lastProgress = curProgress + mu.Unlock() + ul.logTo(fmtStr, ul.filename, lastProgress) + } + }() + + for v := range ch { + mu.Lock() + curProgress = v.Percentage() + mu.Unlock() + } + + ticker.Stop() + + if lastProgress != curProgress { + ul.logTo(fmtStr, ul.filename, curProgress) + } + + if curProgress == 100.0 { + ul.logTo("Uploading of %s has been complete", ul.filename) + } + ul.wg.Done() + }() + return ch +} + +// Wait is waiting for Sink to complete its work, due to it async nature logging messages may be not sequential. +func (ul *uploadLogger) Wait() { + ul.wg.Wait() +} diff --git a/lib/progresslog/progresslog_test.go b/lib/progresslog/progresslog_test.go new file mode 100644 index 0000000000..ab175b6398 --- /dev/null +++ b/lib/progresslog/progresslog_test.go @@ -0,0 +1,88 @@ +package progresslog + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/vmware/govmomi/vim25/progress" +) + +/* +type Report interface { + Percentage() float32 + Detail() string + Error() error +} + +*/ + +type ProgressResults struct { + percentage float32 +} + +func (pr *ProgressResults) Percentage() float32 { + return pr.percentage +} + +func (pr *ProgressResults) Detail() string { + return "" +} + +func (pr *ProgressResults) Error() error { + return nil +} + +var _ progress.Report = &ProgressResults{} + +func TestNewUploadLoggerComplete(t *testing.T) { + var logs []string + logTo := func(format string, args ...interface{}) { + logs = append(logs, fmt.Sprintf(format, args...)) + } + pl := NewUploadLogger(logTo, "unittest", time.Millisecond*10) + progressChan := pl.Sink() + for i := 0; i <= 10; i++ { + res := &ProgressResults{percentage: float32(i * 10)} + progressChan <- res + time.Sleep(time.Duration(time.Millisecond * 5)) + } + close(progressChan) + pl.Wait() + + if assert.True(t, len(logs) > 3) { + last := len(logs) - 1 + assert.Contains(t, logs[0], "unittest") + assert.Contains(t, logs[0], "0.00%") + assert.Contains(t, logs[1], ".00%") + assert.Contains(t, logs[last-1], "100.00%") + assert.Contains(t, logs[last], "complete") + } + +} + +func TestNewUploadLoggerNotComplete(t *testing.T) { + var logs []string + logTo := func(format string, args ...interface{}) { + logs = append(logs, fmt.Sprintf(format, args...)) + } + pl := NewUploadLogger(logTo, "unittest", time.Millisecond*10) + progressChan := pl.Sink() + for i := 0; i < 10; i++ { + res := &ProgressResults{percentage: float32(i * 10)} + progressChan <- res + time.Sleep(time.Duration(time.Millisecond * 5)) + } + close(progressChan) + pl.Wait() + + if assert.True(t, len(logs) > 3) { + last := len(logs) - 1 + assert.Contains(t, logs[0], "unittest") + assert.Contains(t, logs[0], "0.00%") + assert.Contains(t, logs[1], ".00%") + assert.NotContains(t, logs[last], "100.00%") + assert.NotContains(t, logs[last], "complete") + } +} From 676c37ec9fcb78b7bbf3ee8ef1ca2ad660b512ad Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 14:13:10 -0600 Subject: [PATCH 02/10] Format as goimports --- lib/progresslog/progresslog_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/progresslog/progresslog_test.go b/lib/progresslog/progresslog_test.go index ab175b6398..0826e3d54c 100644 --- a/lib/progresslog/progresslog_test.go +++ b/lib/progresslog/progresslog_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/vmware/govmomi/vim25/progress" ) From 881c0a5bc1428c323b9959257b7881b516e4b4f6 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 14:15:43 -0600 Subject: [PATCH 03/10] Make uploadLogger to be a public one --- lib/progresslog/progresslog.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/progresslog/progresslog.go b/lib/progresslog/progresslog.go index 4bb2d9f3e2..cd93d10ce5 100644 --- a/lib/progresslog/progresslog.go +++ b/lib/progresslog/progresslog.go @@ -8,27 +8,27 @@ import ( "github.com/vmware/govmomi/vim25/soap" ) -// UploadParams uses default upload settings as initial input and set uploadLogger as a logger. -func UploadParams(ul *uploadLogger) *soap.Upload { +// UploadParams uses default upload settings as initial input and set UploadLogger as a logger. +func UploadParams(ul *UploadLogger) *soap.Upload { params := soap.DefaultUpload params.Progress = ul return ¶ms } -// uploadLogger io used to track upload progress to ESXi/VC of a specific file. -type uploadLogger struct { +// UploadLogger io used to track upload progress to ESXi/VC of a specific file. +type UploadLogger struct { wg sync.WaitGroup filename string interval time.Duration logTo func(format string, args ...interface{}) } -// NewUploadLogger returns a new instance of uploadLogger. User can provide a logger interface +// NewUploadLogger returns a new instance of UploadLogger. User can provide a logger interface // that will be used to dump output of the current upload status. func NewUploadLogger(logTo func(format string, args ...interface{}), - filename string, progressInterval time.Duration) *uploadLogger { + filename string, progressInterval time.Duration) *UploadLogger { - return &uploadLogger{ + return &UploadLogger{ logTo: logTo, filename: filename, interval: progressInterval, @@ -37,7 +37,7 @@ func NewUploadLogger(logTo func(format string, args ...interface{}), // Sink returns a channel that receives current upload progress status. // Channel has to be closed by the caller. -func (ul *uploadLogger) Sink() chan<- progress.Report { +func (ul *UploadLogger) Sink() chan<- progress.Report { ul.wg.Add(1) ch := make(chan progress.Report) fmtStr := "Uploading %s. Progress: %.2f%%" @@ -81,6 +81,6 @@ func (ul *uploadLogger) Sink() chan<- progress.Report { } // Wait is waiting for Sink to complete its work, due to it async nature logging messages may be not sequential. -func (ul *uploadLogger) Wait() { +func (ul *UploadLogger) Wait() { ul.wg.Wait() } From f66a6f950af9ebfcc880750569678c915c8cac92 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 14:18:23 -0600 Subject: [PATCH 04/10] Add file headers. --- lib/progresslog/progresslog.go | 14 ++++++++++++++ lib/progresslog/progresslog_test.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/lib/progresslog/progresslog.go b/lib/progresslog/progresslog.go index cd93d10ce5..812c8815e1 100644 --- a/lib/progresslog/progresslog.go +++ b/lib/progresslog/progresslog.go @@ -1,3 +1,17 @@ +// Copyright 2018 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package progresslog import ( diff --git a/lib/progresslog/progresslog_test.go b/lib/progresslog/progresslog_test.go index 0826e3d54c..b0ed3ecd9f 100644 --- a/lib/progresslog/progresslog_test.go +++ b/lib/progresslog/progresslog_test.go @@ -1,3 +1,17 @@ +// Copyright 2018 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package progresslog import ( From ec6ad6bf61286295da246b10ad934501d6a002c8 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 15:39:48 -0600 Subject: [PATCH 05/10] Organize imports --- lib/install/management/create.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/install/management/create.go b/lib/install/management/create.go index 8d77dacca7..0fbb652793 100644 --- a/lib/install/management/create.go +++ b/lib/install/management/create.go @@ -22,12 +22,11 @@ import ( "time" "github.com/vmware/govmomi/object" - "github.com/vmware/vic/lib/progresslog" - "github.com/vmware/vic/lib/config" "github.com/vmware/vic/lib/install/data" "github.com/vmware/vic/lib/install/opsuser" "github.com/vmware/vic/lib/install/vchlog" + "github.com/vmware/vic/lib/progresslog" "github.com/vmware/vic/pkg/errors" "github.com/vmware/vic/pkg/retry" "github.com/vmware/vic/pkg/trace" From f31ae1a439b8c48d11b17019ddfcbe53be99f263 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 15:59:20 -0600 Subject: [PATCH 06/10] Improved comments --- lib/progresslog/progresslog.go | 6 +++--- lib/progresslog/progresslog_test.go | 10 ---------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/lib/progresslog/progresslog.go b/lib/progresslog/progresslog.go index 812c8815e1..c23ab4f7e0 100644 --- a/lib/progresslog/progresslog.go +++ b/lib/progresslog/progresslog.go @@ -29,7 +29,7 @@ func UploadParams(ul *UploadLogger) *soap.Upload { return ¶ms } -// UploadLogger io used to track upload progress to ESXi/VC of a specific file. +// UploadLogger is used to track upload progress to ESXi/VC of a specific file. type UploadLogger struct { wg sync.WaitGroup filename string @@ -64,7 +64,7 @@ func (ul *UploadLogger) Sink() chan<- progress.Report { mu := sync.Mutex{} ticker := time.NewTicker(ul.interval) - // Print progress every 3 + // Print progress every ul.interval seconds. go func() { for range ticker.C { mu.Lock() @@ -94,7 +94,7 @@ func (ul *UploadLogger) Sink() chan<- progress.Report { return ch } -// Wait is waiting for Sink to complete its work, due to it async nature logging messages may be not sequential. +// Wait waits for Sink to complete its work, due to its async nature logging messages may be not sequential. func (ul *UploadLogger) Wait() { ul.wg.Wait() } diff --git a/lib/progresslog/progresslog_test.go b/lib/progresslog/progresslog_test.go index b0ed3ecd9f..2826ecf79c 100644 --- a/lib/progresslog/progresslog_test.go +++ b/lib/progresslog/progresslog_test.go @@ -24,15 +24,6 @@ import ( "github.com/vmware/govmomi/vim25/progress" ) -/* -type Report interface { - Percentage() float32 - Detail() string - Error() error -} - -*/ - type ProgressResults struct { percentage float32 } @@ -74,7 +65,6 @@ func TestNewUploadLoggerComplete(t *testing.T) { assert.Contains(t, logs[last-1], "100.00%") assert.Contains(t, logs[last], "complete") } - } func TestNewUploadLoggerNotComplete(t *testing.T) { From e6b95d540053299042d24805afcffbb7a5861924 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 16:00:50 -0600 Subject: [PATCH 07/10] Better error message --- lib/install/management/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/install/management/create.go b/lib/install/management/create.go index 0fbb652793..1986989c98 100644 --- a/lib/install/management/create.go +++ b/lib/install/management/create.go @@ -194,7 +194,7 @@ func (d *Dispatcher) uploadImages(files map[string]string) error { finalMessage = fmt.Sprintf("%s\t\tNote: The VCH will not function without %q...", finalMessage, image) } d.op.Error(finalMessage) - return errors.New("Failed to upload iso images successfully.") + return errors.New("Failed to upload iso images.") } } From 9119f94ea531375bc4fb4caef57caa65ce03004e Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 16:09:04 -0600 Subject: [PATCH 08/10] Uppercases are better --- lib/install/management/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/install/management/create.go b/lib/install/management/create.go index 1986989c98..1b09c41cfc 100644 --- a/lib/install/management/create.go +++ b/lib/install/management/create.go @@ -176,7 +176,7 @@ func (d *Dispatcher) uploadImages(files map[string]string) error { d.op.Warnf("Attempted upload a total of %d times without success, Upload process failed.", uploadRetryLimit) return false } - d.op.Warnf("failed an attempt to upload isos with err (%s), %d retries remain", err.Error(), retryCount) + d.op.Warnf("Failed an attempt to upload isos with err (%s), %d retries remain", err.Error(), retryCount) return true } From 2af566ae3dc48063490b881471f5ae582df1e936 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 16:13:38 -0600 Subject: [PATCH 09/10] better log message --- lib/progresslog/progresslog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/progresslog/progresslog.go b/lib/progresslog/progresslog.go index c23ab4f7e0..1f38b7ce0e 100644 --- a/lib/progresslog/progresslog.go +++ b/lib/progresslog/progresslog.go @@ -87,7 +87,7 @@ func (ul *UploadLogger) Sink() chan<- progress.Report { } if curProgress == 100.0 { - ul.logTo("Uploading of %s has been complete", ul.filename) + ul.logTo("Uploading of %s has been completed", ul.filename) } ul.wg.Done() }() From 829b0bb8107c199e4264c201797910ed54dcea62 Mon Sep 17 00:00:00 2001 From: Volodymyr Burenin Date: Tue, 13 Feb 2018 16:22:12 -0600 Subject: [PATCH 10/10] Moved retry config out of the loop. --- lib/install/management/create.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/install/management/create.go b/lib/install/management/create.go index 1b09c41cfc..a85866d4ff 100644 --- a/lib/install/management/create.go +++ b/lib/install/management/create.go @@ -121,6 +121,12 @@ func (d *Dispatcher) uploadImages(files map[string]string) error { // upload the images d.op.Info("Uploading images for container") + // Build retry config + backoffConf := retry.NewBackoffConfig() + backoffConf.InitialInterval = uploadInitialInterval + backoffConf.MaxInterval = uploadMaxInterval + backoffConf.MaxElapsedTime = uploadMaxElapsedTime + for key, image := range files { baseName := filepath.Base(image) finalMessage := "" @@ -180,12 +186,6 @@ func (d *Dispatcher) uploadImages(files map[string]string) error { return true } - // Build retry config - backoffConf := retry.NewBackoffConfig() - backoffConf.InitialInterval = uploadInitialInterval - backoffConf.MaxInterval = uploadMaxInterval - backoffConf.MaxElapsedTime = uploadMaxElapsedTime - uploadErr := retry.DoWithConfig(operationForRetry, uploadRetryDecider, backoffConf) if uploadErr != nil { finalMessage = fmt.Sprintf("\t\tUpload failed for %q: %s\n", image, uploadErr)