backup: store backupmeta to s3

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	cloud.google.com/go/storage v1.4.0
+	github.com/aws/aws-sdk-go v1.26.1
 	github.com/cheggaaa/pb/v3 v3.0.1
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect

go.sum

@@ -27,6 +27,8 @@ github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmx
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.26.1 h1:JGQggXhOiNJIqsmbYUl3cYtJZUffeOWlHtxfzGK7WPI=
+github.com/aws/aws-sdk-go v1.26.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -171,6 +173,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=

pkg/backup/client.go

@@ -107,9 +107,11 @@ func (bc *Client) SetStorage(ctx context.Context, backend *backup.StorageBackend
 		return err
 	}
 	// backupmeta already exists
-	if exist, err := bc.storage.FileExists(ctx, utils.MetaFile); err != nil {
-		return err
-	} else if exist {
+	exist, err := bc.storage.FileExists(ctx, utils.MetaFile)
+	if err != nil {
+		return errors.Annotatef(err, "error occurred when checking %s file", utils.MetaFile)
+	}
+	if exist {
 		return errors.New("backup meta exists, may be some backup files in the path already")
 	}
 	bc.backend = backend

pkg/storage/flags.go

@@ -6,8 +6,15 @@ import (
 	"github.com/spf13/pflag"
 )
 
+const (
+	// flagSendCredentialOption specify whether to send credentials to tikv
+	flagSendCredentialOption = "send-credentials-to-tikv"
+)
+
 // DefineFlags adds flags to the flag set corresponding to all backend options.
 func DefineFlags(flags *pflag.FlagSet) {
+	flags.BoolP(flagSendCredentialOption, "c", true,
+		"Whether send credentials to tikv")
 	defineS3Flags(flags)
 	defineGCSFlags(flags)
 }

pkg/storage/parse.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"net/url"
+	"strings"
 
 	"github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/backup"
@@ -38,7 +39,11 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken
 		return &backup.StorageBackend{Backend: &backup.StorageBackend_Noop{Noop: noop}}, nil
 
 	case "s3":
-		s3 := &backup.S3{Bucket: u.Host, Prefix: u.Path}
+		if u.Host == "" {
+			return nil, errors.Errorf("please specify the bucket for s3 in %s", rawURL)
+		}
+		prefix := strings.Trim(u.Path, "/")
+		s3 := &backup.S3{Bucket: u.Host, Prefix: prefix}
 		if options != nil {
 			if err := options.S3.apply(s3); err != nil {
 				return nil, err

pkg/storage/parse_test.go

@@ -1,4 +1,4 @@
-package storage_test
+package storage
 
 import (
 	"io/ioutil"
@@ -7,8 +7,6 @@ import (
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/kvproto/pkg/backup"
-
-	"github.com/pingcap/br/pkg/storage"
 )
 
 func Test(t *testing.T) {
@@ -20,38 +18,38 @@ type testStorageSuite struct{}
 var _ = Suite(&testStorageSuite{})
 
 func (r *testStorageSuite) TestCreateStorage(c *C) {
-	_, err := storage.ParseBackend("1invalid:", nil)
+	_, err := ParseBackend("1invalid:", nil)
 	c.Assert(err, ErrorMatches, "parse 1invalid:: first path segment in URL cannot contain colon")
 
-	_, err = storage.ParseBackend("net:storage", nil)
+	_, err = ParseBackend("net:storage", nil)
 	c.Assert(err, ErrorMatches, "storage net not support yet")
 
-	s, err := storage.ParseBackend("local:///tmp/storage", nil)
+	s, err := ParseBackend("local:///tmp/storage", nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.GetLocal().GetPath(), Equals, "/tmp/storage")
 
-	s, err = storage.ParseBackend("file:///tmp/storage", nil)
+	s, err = ParseBackend("file:///tmp/storage", nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.GetLocal().GetPath(), Equals, "/tmp/storage")
 
-	s, err = storage.ParseBackend("noop://", nil)
+	s, err = ParseBackend("noop://", nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.GetNoop(), NotNil)
 
-	_, err = storage.ParseBackend("s3://bucket/more/prefix/", &storage.BackendOptions{})
-	c.Assert(err, ErrorMatches, `must provide either 's3\.region' or 's3\.endpoint'`)
+	_, err = ParseBackend("s3:///bucket/more/prefix/", &BackendOptions{})
+	c.Assert(err, ErrorMatches, `please specify the bucket for s3 in s3:///bucket/more/prefix/`)
 
-	s3opt := &storage.BackendOptions{
-		S3: storage.S3BackendOptions{
+	s3opt := &BackendOptions{
+		S3: S3BackendOptions{
 			Endpoint: "https://s3.example.com/",
 		},
 	}
-	s, err = storage.ParseBackend("s3://bucket2/prefix/", s3opt)
+	s, err = ParseBackend("s3://bucket2/prefix/", s3opt)
 	c.Assert(err, IsNil)
 	s3 := s.GetS3()
 	c.Assert(s3, NotNil)
 	c.Assert(s3.Bucket, Equals, "bucket2")
-	c.Assert(s3.Prefix, Equals, "/prefix/")
+	c.Assert(s3.Prefix, Equals, "prefix")
 	c.Assert(s3.Endpoint, Equals, "https://s3.example.com/")
 
 	fakeCredentialsFile, err := ioutil.TempFile("", "fakeCredentialsFile")
@@ -60,20 +58,20 @@ func (r *testStorageSuite) TestCreateStorage(c *C) {
 		fakeCredentialsFile.Close()
 		os.Remove(fakeCredentialsFile.Name())
 	}()
-	gcsOpt := &storage.BackendOptions{
-		GCS: storage.GCSBackendOptions{
+	gcsOpt := &BackendOptions{
+		GCS: GCSBackendOptions{
 			Endpoint:        "https://gcs.example.com/",
 			CredentialsFile: fakeCredentialsFile.Name(),
 		},
 	}
-	s, err = storage.ParseBackend("gcs://bucket2/prefix/", gcsOpt)
+	s, err = ParseBackend("gcs://bucket2/prefix/", gcsOpt)
 	c.Assert(err, IsNil)
 	gcs := s.GetGcs()
 	c.Assert(gcs, NotNil)
 	c.Assert(gcs.Bucket, Equals, "bucket2")
 	c.Assert(gcs.Prefix, Equals, "prefix/")
 	c.Assert(gcs.Endpoint, Equals, "https://gcs.example.com/")
-	s, err = storage.ParseBackend("gcs://bucket/more/prefix/", gcsOpt)
+	s, err = ParseBackend("gcs://bucket/more/prefix/", gcsOpt)
 	c.Assert(err, IsNil)
 	gcs = s.GetGcs()
 	c.Assert(gcs, NotNil)
@@ -83,21 +81,21 @@ func (r *testStorageSuite) TestCreateStorage(c *C) {
 }
 
 func (r *testStorageSuite) TestFormatBackendURL(c *C) {
-	url := storage.FormatBackendURL(&backup.StorageBackend{
+	url := FormatBackendURL(&backup.StorageBackend{
 		Backend: &backup.StorageBackend_Local{
 			Local: &backup.Local{Path: "/tmp/file"},
 		},
 	})
 	c.Assert(url.String(), Equals, "local:///tmp/file")
 
-	url = storage.FormatBackendURL(&backup.StorageBackend{
+	url = FormatBackendURL(&backup.StorageBackend{
 		Backend: &backup.StorageBackend_Noop{
 			Noop: &backup.Noop{},
 		},
 	})
 	c.Assert(url.String(), Equals, "noop:///")
 
-	url = storage.FormatBackendURL(&backup.StorageBackend{
+	url = FormatBackendURL(&backup.StorageBackend{
 		Backend: &backup.StorageBackend_S3{
 			S3: &backup.S3{
 				Bucket:   "bucket",
@@ -108,7 +106,7 @@ func (r *testStorageSuite) TestFormatBackendURL(c *C) {
 	})
 	c.Assert(url.String(), Equals, "s3://bucket/some%20prefix/")
 
-	url = storage.FormatBackendURL(&backup.StorageBackend{
+	url = FormatBackendURL(&backup.StorageBackend{
 		Backend: &backup.StorageBackend_Gcs{
 			Gcs: &backup.GCS{
 				Bucket:   "bucket",