From 75af808739e18b70daf5e1b076fc9d7e7514fa56 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:13:26 +0800 Subject: [PATCH 1/9] refactor(cloudreve): use retry-go for uploads --- drivers/cloudreve/util.go | 216 ++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 112 deletions(-) diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index ef37811ad..45f558086 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/cookie" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -240,8 +241,6 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) - retryCount := 0 - maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() @@ -255,53 +254,48 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U if err != nil { return err } - req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), - driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + err = retry.Do( + func() error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req.ContentLength = byteSize + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return fmt.Errorf("server error: %d", res.StatusCode) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Authorization", fmt.Sprint(credential)) - req.Header.Set("User-Agent", d.getUA()) - err = func() error { - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return errors.New(res.Status) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return err - } - var up Resp - err = json.Unmarshal(body, &up) - if err != nil { - return err - } - if up.Code != 0 { - return errors.New(up.Msg) - } - return nil - }() - if err == nil { - retryCount = 0 - finish += byteSize - up(float64(finish) * 100 / float64(stream.GetSize())) - chunk++ - } else { - retryCount++ - if retryCount > maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) - } - backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1< maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1<") - req, err := http.NewRequest( - "POST", + req, err := http.NewRequestWithContext(ctx, + http.MethodPost, u.CompleteURL, strings.NewReader(bodyBuilder.String()), ) From 7ccd3c805e6258f2e9936dcfe41405106ec85375 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:13:31 +0800 Subject: [PATCH 2/9] refactor(cloudreve_v4): use retry-go for uploads --- drivers/cloudreve_v4/util.go | 217 +++++++++++++++++------------------ 1 file changed, 106 insertions(+), 111 deletions(-) diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 2c7d041e4..0e11c8d6b 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -258,8 +259,6 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) - retryCount := 0 - maxRetries := 3 for finish < file.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() @@ -273,53 +272,48 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F if err != nil { return err } - req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), - driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + err = retry.Do( + func() error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + + req.ContentLength = byteSize + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return fmt.Errorf("server error: %d", res.StatusCode) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }, retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Authorization", fmt.Sprint(credential)) - req.Header.Set("User-Agent", d.getUA()) - err = func() error { - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return errors.New(res.Status) - } - body, err := io.ReadAll(res.Body) - if err != nil { - return err - } - var up Resp - err = json.Unmarshal(body, &up) - if err != nil { - return err - } - if up.Code != 0 { - return errors.New(up.Msg) - } - return nil - }() - if err == nil { - retryCount = 0 - finish += byteSize - up(float64(finish) * 100 / float64(file.GetSize())) - chunk++ - } else { - retryCount++ - if retryCount > maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) - } - backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1< maxRetries { - return fmt.Errorf("upload failed after %d retries due to server errors", maxRetries) - } - backoff := time.Duration(1<") - req, err := http.NewRequest( - "POST", + req, err := http.NewRequestWithContext(ctx, + http.MethodPost, u.CompleteURL, strings.NewReader(bodyBuilder.String()), ) From 701a6887d4eb81c09a22eecf94fdb3a3c253f750 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:13:37 +0800 Subject: [PATCH 3/9] refactor(onedrive): use retry-go for uploads --- drivers/onedrive/util.go | 66 ++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 2093bffbf..d0ef5e3ce 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -230,7 +231,7 @@ func toAPIMetadata(stream model.FileStreamer) Metadata { func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/createUploadSession" - metadata := map[string]interface{}{"item": toAPIMetadata(stream)} + metadata := map[string]any{"item": toAPIMetadata(stream)} res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(metadata).SetContext(ctx) }, nil) @@ -240,8 +241,6 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 - retryCount := 0 - maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() @@ -255,39 +254,40 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + err = retry.Do( + func() error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req.ContentLength = byteSize + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1< Date: Mon, 21 Jul 2025 15:13:48 +0800 Subject: [PATCH 4/9] refactor(onedrive_app): use retry-go for uploads --- drivers/onedrive_app/util.go | 64 ++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 031a11367..ac5449c5b 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -154,8 +155,6 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 - retryCount := 0 - maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() @@ -169,39 +168,40 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + err = retry.Do( + func() error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req.ContentLength = byteSize + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + return fmt.Errorf("server error: %d", res.StatusCode) + case res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200: + data, _ := io.ReadAll(res.Body) + return errors.New(string(data)) + default: + return nil + } + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) if err != nil { return err } - req = req.WithContext(ctx) - req.ContentLength = byteSize - // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) - req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } - // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - switch { - case res.StatusCode >= 500 && res.StatusCode <= 504: - retryCount++ - if retryCount > maxRetries { - res.Body.Close() - return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) - } - backoff := time.Duration(1< Date: Mon, 21 Jul 2025 15:15:05 +0800 Subject: [PATCH 5/9] chore(onedrive_app): remove unnecessary error handling for host retrieval --- drivers/onedrive_app/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index ac5449c5b..8069934fa 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -41,7 +41,7 @@ var onedriveHostMap = map[string]Host{ } func (d *OnedriveAPP) GetMetaUrl(auth bool, path string) string { - host, _ := onedriveHostMap[d.Region] + host := onedriveHostMap[d.Region] path = utils.EncodePath(path, true) if auth { return host.Oauth From 87f4a424dc2fdffd105bab523d28e7cb6fd6752f Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:43:46 +0800 Subject: [PATCH 6/9] feat(cloudreve): move read logic inside retry block --- drivers/cloudreve/util.go | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 45f558086..06b51319a 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -247,15 +247,15 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } - err = retry.Do( + err := retry.Do( func() error { + utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { @@ -310,15 +310,15 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[Cloudreve-OneDrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } - err = retry.Do( + err := retry.Do( func() error { + utils.Log.Debugf("[Cloudreve-OneDrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { @@ -369,15 +369,15 @@ func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u Uploa } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[Cloudreve-S3] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } - err = retry.Do( + err := retry.Do( func() error { + utils.Log.Debugf("[Cloudreve-S3] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.UploadURLs[chunk], driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) if err != nil { From c96ae1530d1bd776d42d40a911febc6b0833c393 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:44:15 +0800 Subject: [PATCH 7/9] feat(cloudreve_v4): move read logic inside retry block --- drivers/cloudreve_v4/util.go | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 0e11c8d6b..68a89930f 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -265,15 +265,15 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F } left := file.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(file, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } - err = retry.Do( + err := retry.Do( func() error { + utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"?chunk="+strconv.Itoa(chunk), driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { @@ -328,15 +328,15 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u } left := file.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[CloudreveV4-OneDrive] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(file, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } - err = retry.Do( + err := retry.Do( func() error { + utils.Log.Debugf("[CloudreveV4-OneDrive] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { @@ -388,15 +388,15 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU } left := file.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[CloudreveV4-S3] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(file, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } - err = retry.Do( + err := retry.Do( func() error { + utils.Log.Debugf("[CloudreveV4-S3] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.UploadUrls[chunk], driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) if err != nil { From afe01ddb7738b5d21ad2bb6cd636b030f70f4c1f Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:44:38 +0800 Subject: [PATCH 8/9] feat(onedrive): move read logic inside retry block --- drivers/onedrive/util.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index d0ef5e3ce..be86d50b9 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -247,15 +247,15 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } err = retry.Do( func() error { + utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { From 4f88eaa289dcba772f12f5c5e238183561faaa84 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 21 Jul 2025 15:44:44 +0800 Subject: [PATCH 9/9] feat(onedrive_app): move read logic inside retry block --- drivers/onedrive_app/util.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 8069934fa..3be82b5ae 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -161,15 +161,15 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. } left := stream.GetSize() - finish byteSize := min(left, DEFAULT) - utils.Log.Debugf("[OnedriveAPP] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) - byteData := make([]byte, byteSize) - n, err := io.ReadFull(stream, byteData) - utils.Log.Debug(err, n) - if err != nil { - return err - } err = retry.Do( func() error { + utils.Log.Debugf("[OnedriveAPP] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil {