From 1b41ead7c3bc67fc1f8ba5ebf96eb10e83a03291 Mon Sep 17 00:00:00 2001 From: luancheng Date: Mon, 3 Feb 2020 21:08:36 +0800 Subject: [PATCH 1/5] backup: add backupts file as lock --- cmd/backup.go | 12 ++++++++++++ pkg/backup/client.go | 18 ++++++++++++++++++ pkg/utils/schema.go | 2 ++ 3 files changed, 32 insertions(+) diff --git a/cmd/backup.go b/cmd/backup.go index 73ae6106f..2008bf582 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "fmt" "github.com/pingcap/errors" "github.com/pingcap/log" @@ -108,6 +109,11 @@ func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error { return err } + err = client.SaveBackupTS(ctx, backupTS) + if err != nil { + return err + } + defer summary.Summary(cmdName) ranges, backupSchemas, err := backup.BuildBackupRangeAndSchema( @@ -170,6 +176,12 @@ func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error { if err != nil { return err } + + content, err := client.GetBackupTS(ctx) + if err != nil { + return err + } + fmt.Println(string(content)) return nil } diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 5cba2d9bf..38300fd76 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "io" "sync" "time" @@ -120,6 +121,23 @@ func (bc *Client) SetStorage(ctx context.Context, backend *backup.StorageBackend return nil } +// SaveBackupTS saves the backup ts at the given path. +func (bc *Client) SaveBackupTS(ctx context.Context, backupTS uint64) error { + backendURL := storage.FormatBackendURL(bc.backend) + log.Info("save backup ts", zap.Stringer("path", &backendURL)) + fileString := fmt.Sprintf( + "**************************************" + + "\nStarted backup at: %v\nBackupTS: %d\n" + + "**************************************", + time.Now().Format("2006-01-02 15:04:05"), backupTS) + return bc.storage.Write(ctx, utils.TSFile, []byte(fileString)) +} + +// GetBackupTS get the backup ts from given path. +func (bc *Client) GetBackupTS(ctx context.Context) ([]byte, error) { + return bc.storage.Read(ctx, utils.TSFile) +} + // SaveBackupMeta saves the current backup meta at the given path. func (bc *Client) SaveBackupMeta(ctx context.Context) error { backupMetaData, err := proto.Marshal(&bc.backupMeta) diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index 67d28132f..6d4db4bd2 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -17,6 +17,8 @@ import ( const ( // MetaFile represents file name MetaFile = "backupmeta" + // TSFile + TSFile = "backupts" // MetaJSONFile represents backup meta json file name MetaJSONFile = "backupmeta.json" ) From bade240f2f0b666bc1a6b54b451dc8d9b46031b6 Mon Sep 17 00:00:00 2001 From: luancheng Date: Tue, 4 Feb 2020 15:41:09 +0800 Subject: [PATCH 2/5] address comment --- pkg/utils/schema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/schema.go b/pkg/utils/schema.go index 6d4db4bd2..9ade83d57 100644 --- a/pkg/utils/schema.go +++ b/pkg/utils/schema.go @@ -17,7 +17,7 @@ import ( const ( // MetaFile represents file name MetaFile = "backupmeta" - // TSFile + // TSFile represents backup ts file, it generated before backup TSFile = "backupts" // MetaJSONFile represents backup meta json file name MetaJSONFile = "backupmeta.json" From da5cbb8c0746339020fd62878cd23a9b0ff0e97d Mon Sep 17 00:00:00 2001 From: luancheng Date: Tue, 4 Feb 2020 17:33:31 +0800 Subject: [PATCH 3/5] fix ci --- pkg/backup/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 38300fd76..799e93977 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -126,8 +126,8 @@ func (bc *Client) SaveBackupTS(ctx context.Context, backupTS uint64) error { backendURL := storage.FormatBackendURL(bc.backend) log.Info("save backup ts", zap.Stringer("path", &backendURL)) fileString := fmt.Sprintf( - "**************************************" + - "\nStarted backup at: %v\nBackupTS: %d\n" + + "**************************************"+ + "\nStarted backup at: %v\nBackupTS: %d\n"+ "**************************************", time.Now().Format("2006-01-02 15:04:05"), backupTS) return bc.storage.Write(ctx, utils.TSFile, []byte(fileString)) From 6dd158f62ff004d02e454e6199d3c2d12b891472 Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 5 Feb 2020 21:38:36 +0800 Subject: [PATCH 4/5] fix ci --- go.mod | 1 + go.sum | 3 +++ pkg/backup/client.go | 25 ++++++++++++++++++------- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 9951c2922..a0169a4c0 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834 github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 github.com/prometheus/client_golang v1.0.0 + github.com/prometheus/common v0.4.1 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index 085e00355..56c7af7ef 100644 --- a/go.sum +++ b/go.sum @@ -24,7 +24,9 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.26.1 h1:JGQggXhOiNJIqsmbYUl3cYtJZUffeOWlHtxfzGK7WPI= @@ -567,6 +569,7 @@ google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/backup/client.go b/pkg/backup/client.go index 799e93977..9fc0e5316 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "encoding/json" - "fmt" "io" "sync" "time" @@ -123,14 +122,26 @@ func (bc *Client) SetStorage(ctx context.Context, backend *backup.StorageBackend // SaveBackupTS saves the backup ts at the given path. func (bc *Client) SaveBackupTS(ctx context.Context, backupTS uint64) error { + if exists, _ := bc.storage.FileExists(ctx, utils.TSFile); exists { + return errors.New("this storage has backup file, consider clear this storage or change to new one") + } backendURL := storage.FormatBackendURL(bc.backend) + log.Info("save backup ts", zap.Stringer("path", &backendURL)) - fileString := fmt.Sprintf( - "**************************************"+ - "\nStarted backup at: %v\nBackupTS: %d\n"+ - "**************************************", - time.Now().Format("2006-01-02 15:04:05"), backupTS) - return bc.storage.Write(ctx, utils.TSFile, []byte(fileString)) + type fileContent struct { + StartedAt string `json:"started_at"` + BackupTS uint64 `json:"backup_ts"` + } + f := &fileContent{ + StartedAt: time.Now().Format("2006-01-02 15:04:05"), + BackupTS: backupTS, + } + fileBytes, err := json.MarshalIndent(f, "", "\t") + if err != nil { + return err + } + + return bc.storage.Write(ctx, utils.TSFile, fileBytes) } // GetBackupTS get the backup ts from given path. From 796b8ed947f08ee7794b0116bc540052ca7e4010 Mon Sep 17 00:00:00 2001 From: luancheng Date: Wed, 5 Feb 2020 21:48:45 +0800 Subject: [PATCH 5/5] fix ci --- go.mod | 1 - go.sum | 3 --- 2 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index a0169a4c0..9951c2922 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834 github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33 github.com/prometheus/client_golang v1.0.0 - github.com/prometheus/common v0.4.1 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index 56c7af7ef..085e00355 100644 --- a/go.sum +++ b/go.sum @@ -24,9 +24,7 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.26.1 h1:JGQggXhOiNJIqsmbYUl3cYtJZUffeOWlHtxfzGK7WPI= @@ -569,7 +567,6 @@ google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20180810215634-df19058c872c/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=