Skip to content
This repository was archived by the owner on Jul 24, 2024. It is now read-only.

storage: support GCS backup and restore #102

Merged
overvenus merged 15 commits into from
Dec 13, 2019
Merged

storage: support GCS backup and restore #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
df6b8c8
storage: support GCS backup and restore
SunRunAway Dec 11, 2019
c2f254d
update dependency
SunRunAway Dec 11, 2019
4d4212e
update flag usage message
SunRunAway Dec 11, 2019
266501d
go fmt
SunRunAway Dec 11, 2019
01f5dd1
update flag usage message
SunRunAway Dec 11, 2019
4552100
fix & add test
SunRunAway Dec 11, 2019
7a8efe1
go fmt
SunRunAway Dec 11, 2019
778ab12
make golint happy
SunRunAway Dec 12, 2019
8843018
check bucket exists
SunRunAway Dec 12, 2019
2148056
add gcs_test
SunRunAway Dec 12, 2019
21fdd37
update dependency
SunRunAway Dec 12, 2019
d6fda29
update test
SunRunAway Dec 12, 2019
b9d2053
use ctx when newGCSStorage
SunRunAway Dec 12, 2019
a98e517
fix conflict
SunRunAway Dec 12, 2019
2460ff3
wrap help message
SunRunAway Dec 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func newFullBackupCommand() *cobra.Command {
return err
}

err = client.SetStorage(u)
err = client.SetStorage(ctx, u)
if err != nil {
return err
}
Expand Down Expand Up @@ -171,7 +171,7 @@ func newFullBackupCommand() *cobra.Command {
// Checksum has finished
close(updateCh)

return client.SaveBackupMeta()
return client.SaveBackupMeta(ctx)
},
}
return command
Expand Down Expand Up @@ -201,7 +201,7 @@ func newTableBackupCommand() *cobra.Command {
return err
}

err = client.SetStorage(u)
err = client.SetStorage(ctx, u)
if err != nil {
return err
}
Expand Down Expand Up @@ -306,7 +306,7 @@ func newTableBackupCommand() *cobra.Command {
// Checksum has finished
close(updateCh)

return client.SaveBackupMeta()
return client.SaveBackupMeta(ctx)
},
}
command.Flags().StringP("db", "", "", "backup a table in the specific db")
Expand Down
12 changes: 6 additions & 6 deletions cmd/restore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func newFullRestoreCommand() *cobra.Command {
return errors.Trace(err)
}
defer client.Close()
err = initRestoreClient(client, cmd.Flags())
err = initRestoreClient(ctx, client, cmd.Flags())
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -181,7 +181,7 @@ func newDbRestoreCommand() *cobra.Command {
return errors.Trace(err)
}
defer client.Close()
err = initRestoreClient(client, cmd.Flags())
err = initRestoreClient(ctx, client, cmd.Flags())
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -291,7 +291,7 @@ func newTableRestoreCommand() *cobra.Command {
return errors.Trace(err)
}
defer client.Close()
err = initRestoreClient(client, cmd.Flags())
err = initRestoreClient(ctx, client, cmd.Flags())
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -387,7 +387,7 @@ func newTableRestoreCommand() *cobra.Command {
return command
}

func initRestoreClient(client *restore.Client, flagSet *flag.FlagSet) error {
func initRestoreClient(ctx context.Context, client *restore.Client, flagSet *flag.FlagSet) error {
u, err := storage.ParseBackendFromFlags(flagSet, FlagStorage)
if err != nil {
return err
Expand All @@ -397,11 +397,11 @@ func initRestoreClient(client *restore.Client, flagSet *flag.FlagSet) error {
return err
}
client.SetRateLimit(rateLimit * utils.MB)
s, err := storage.Create(u)
s, err := storage.Create(ctx, u)
if err != nil {
return errors.Trace(err)
}
metaData, err := s.Read(utils.MetaFile)
metaData, err := s.Read(ctx, utils.MetaFile)
if err != nil {
return errors.Trace(err)
}
Expand Down
17 changes: 12 additions & 5 deletions cmd/validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
Expand Down Expand Up @@ -48,16 +49,19 @@ func newCheckSumCommand() *cobra.Command {
Use: "checksum",
Short: "check the backup data",
RunE: func(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithCancel(GetDefaultContext())
defer cancel()

u, err := storage.ParseBackendFromFlags(cmd.Flags(), FlagStorage)
if err != nil {
return err
}
s, err := storage.Create(u)
s, err := storage.Create(ctx, u)
if err != nil {
return errors.Trace(err)
}

metaData, err := s.Read(utils.MetaFile)
metaData, err := s.Read(ctx, utils.MetaFile)
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -105,7 +109,7 @@ func newCheckSumCommand() *cobra.Command {
)

var data []byte
data, err = s.Read(file.Name)
data, err = s.Read(ctx, file.Name)
if err != nil {
return errors.Trace(err)
}
Expand Down Expand Up @@ -138,6 +142,9 @@ func newBackupMetaCommand() *cobra.Command {
Use: "backupmeta",
Short: "check the backup meta",
RunE: func(cmd *cobra.Command, _ []string) error {
ctx, cancel := context.WithCancel(GetDefaultContext())
defer cancel()

tableIDOffset, err := cmd.Flags().GetUint64("offset")
if err != nil {
return err
Expand All @@ -146,12 +153,12 @@ func newBackupMetaCommand() *cobra.Command {
if err != nil {
return err
}
s, err := storage.Create(u)
s, err := storage.Create(ctx, u)
if err != nil {
log.Error("create storage failed", zap.Error(err))
return errors.Trace(err)
}
data, err := s.Read(utils.MetaFile)
data, err := s.Read(ctx, utils.MetaFile)
if err != nil {
log.Error("load backupmeta failed", zap.Error(err))
return err
Expand Down
14 changes: 6 additions & 8 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,39 +3,37 @@ module github.com/pingcap/br
go 1.13

require (
cloud.google.com/go/storage v1.4.0
github.com/cheggaaa/pb/v3 v3.0.1
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
github.com/fsouza/fake-gcs-server v1.15.0
github.com/go-sql-driver/mysql v1.4.1
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
github.com/google/btree v1.0.0
github.com/google/uuid v1.1.1
github.com/onsi/ginkgo v1.10.3 // indirect
github.com/onsi/gomega v1.7.1 // indirect
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
github.com/pingcap/errors v0.11.4
github.com/pingcap/kvproto v0.0.0-20191210040729-c23886becb54
github.com/pingcap/kvproto v0.0.0-20191212073621-373b0fec09a1
github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
github.com/pingcap/parser v0.0.0-20191205054626-288fe5207ce6
github.com/pingcap/pd v1.1.0-beta.0.20191115131715-6b7dc037010e
github.com/pingcap/tidb v1.1.0-beta.0.20191205065313-6083b21f986b
github.com/pingcap/tidb-tools v3.0.8-0.20191209062450-c67149676f5c+incompatible
github.com/pingcap/tipb v0.0.0-20191126033718-169898888b24
github.com/prometheus/client_golang v1.0.0
github.com/sirupsen/logrus v1.2.0
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.3
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 // indirect
go.opencensus.io v0.22.2 // indirect
go.uber.org/zap v1.10.0
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
golang.org/x/net v0.0.0-20191011234655-491137f69257 // indirect
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a // indirect
google.golang.org/genproto v0.0.0-20190905072037-92dd089d5514 // indirect
google.golang.org/api v0.14.0
google.golang.org/grpc v1.24.0
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
)

replace github.com/golang/lint => golang.org/x/lint v0.0.0-20190930215403-16217165b5de
Loading