storage: S3 test refactoring and fix some bugs

go.mod

@@ -3,13 +3,14 @@ module github.com/pingcap/br
 go 1.13
 
 require (
-	cloud.google.com/go/storage v1.5.0
-	github.com/aws/aws-sdk-go v1.30.24
+	cloud.google.com/go/storage v1.6.0
+	github.com/aws/aws-sdk-go v1.35.2
 	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/coreos/go-semver v0.3.0
-	github.com/fsouza/fake-gcs-server v1.17.0
+	github.com/fsouza/fake-gcs-server v1.19.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gogo/protobuf v1.3.1
+	github.com/golang/mock v1.4.4
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
@@ -27,14 +28,14 @@ require (
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09
-	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
+	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
 	go.uber.org/multierr v1.5.0
 	go.uber.org/zap v1.16.0
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73 // indirect
-	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f // indirect
 	golang.org/x/tools v0.0.0-20200904185747-39188db58858 // indirect
-	google.golang.org/api v0.15.1
-	google.golang.org/grpc v1.26.0
+	google.golang.org/api v0.22.0
+	google.golang.org/grpc v1.27.1
 )

pkg/storage/gcs.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
-	"net/http"
 
 	"cloud.google.com/go/storage"
 	"github.com/pingcap/errors"
@@ -152,23 +151,14 @@ func (s *gcsStorage) CreateUploader(ctx context.Context, name string) (Uploader,
 	panic("gcs storage not support multi-upload")
 }
 
-func newGCSStorage(ctx context.Context, gcs *backup.GCS, sendCredential bool) (*gcsStorage, error) {
-	return newGCSStorageWithHTTPClient(ctx, gcs, nil, sendCredential)
-}
-
-func newGCSStorageWithHTTPClient( // revive:disable-line:flag-parameter
-	ctx context.Context,
-	gcs *backup.GCS,
-	hclient *http.Client,
-	sendCredential bool,
-) (*gcsStorage, error) {
+func newGCSStorage(ctx context.Context, gcs *backup.GCS, opts *ExternalStorageOptions) (*gcsStorage, error) {
 	var clientOps []option.ClientOption
 	if gcs.CredentialsBlob == "" {
 		creds, err := google.FindDefaultCredentials(ctx, storage.ScopeReadWrite)
 		if err != nil {
 			return nil, errors.Annotatef(berrors.ErrStorageInvalidConfig, "%v Or you should provide '--gcs.credentials_file'", err)
 		}
-		if sendCredential {
+		if opts.SendCredentials {
 			if len(creds.JSON) > 0 {
 				gcs.CredentialsBlob = string(creds.JSON)
 			} else {
@@ -184,24 +174,26 @@ func newGCSStorageWithHTTPClient( // revive:disable-line:flag-parameter
 	if gcs.Endpoint != "" {
 		clientOps = append(clientOps, option.WithEndpoint(gcs.Endpoint))
 	}
-	if hclient != nil {
-		clientOps = append(clientOps, option.WithHTTPClient(hclient))
+	if opts.HTTPClient != nil {
+		clientOps = append(clientOps, option.WithHTTPClient(opts.HTTPClient))
 	}
 	client, err := storage.NewClient(ctx, clientOps...)
 	if err != nil {
 		return nil, err
 	}
 
-	if !sendCredential {
+	if !opts.SendCredentials {
 		// Clear the credentials if exists so that they will not be sent to TiKV
 		gcs.CredentialsBlob = ""
 	}
 
 	bucket := client.Bucket(gcs.Bucket)
-	// check bucket exists
-	_, err = bucket.Attrs(ctx)
-	if err != nil {
-		return nil, err
+	if !opts.SkipCheckPath {
+		// check bucket exists
+		_, err = bucket.Attrs(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return &gcsStorage{gcs: gcs, bucket: bucket}, nil
 }

pkg/storage/gcs_test.go

@@ -30,7 +30,11 @@ func (r *testStorageSuite) TestGCS(c *C) {
 		PredefinedAcl:   "private",
 		CredentialsBlob: "Fake Credentials",
 	}
-	stg, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), false)
+	stg, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+		SendCredentials: false,
+		SkipCheckPath:   false,
+		HTTPClient:      server.HTTPClient(),
+	})
 	c.Assert(err, IsNil)
 
 	err = stg.Write(ctx, "key", []byte("data"))
@@ -77,7 +81,11 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) {
 			PredefinedAcl:   "private",
 			CredentialsBlob: "FakeCredentials",
 		}
-		_, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), true)
+		_, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+			SendCredentials: true,
+			SkipCheckPath:   false,
+			HTTPClient:      server.HTTPClient(),
+		})
 		c.Assert(err, IsNil)
 		c.Assert(gcs.CredentialsBlob, Equals, "FakeCredentials")
 	}
@@ -90,7 +98,11 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) {
 			PredefinedAcl:   "private",
 			CredentialsBlob: "FakeCredentials",
 		}
-		_, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), false)
+		_, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+			SendCredentials: false,
+			SkipCheckPath:   false,
+			HTTPClient:      server.HTTPClient(),
+		})
 		c.Assert(err, IsNil)
 		c.Assert(gcs.CredentialsBlob, Equals, "")
 	}
@@ -115,7 +127,11 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) {
 			PredefinedAcl:   "private",
 			CredentialsBlob: "",
 		}
-		_, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), true)
+		_, err = newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+			SendCredentials: true,
+			SkipCheckPath:   false,
+			HTTPClient:      server.HTTPClient(),
+		})
 		c.Assert(err, IsNil)
 		c.Assert(gcs.CredentialsBlob, Equals, `{"type": "service_account"}`)
 	}
@@ -140,7 +156,11 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) {
 			PredefinedAcl:   "private",
 			CredentialsBlob: "",
 		}
-		_, err = newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), false)
+		_, err = newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+			SendCredentials: false,
+			SkipCheckPath:   false,
+			HTTPClient:      server.HTTPClient(),
+		})
 		c.Assert(err, IsNil)
 		c.Assert(gcs.CredentialsBlob, Equals, "")
 	}
@@ -154,7 +174,11 @@ func (r *testStorageSuite) TestNewGCSStorage(c *C) {
 			PredefinedAcl:   "private",
 			CredentialsBlob: "",
 		}
-		_, err := newGCSStorageWithHTTPClient(ctx, gcs, server.HTTPClient(), true)
+		_, err := newGCSStorage(ctx, gcs, &ExternalStorageOptions{
+			SendCredentials: true,
+			SkipCheckPath:   false,
+			HTTPClient:      server.HTTPClient(),
+		})
 		c.Assert(err, NotNil)
 	}
 }