Cherry-pick to v3.1.0.beta2

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,29 @@
+---
+name: "🐛 Bug Report"
+about: Something isn't working as expected
+title: ''
+labels: 'bug'
+---
+
+Please answer these questions before submitting your issue. Thanks!
+
+1. What did you do?
+If possible, provide a recipe for reproducing the error.
+
+
+2. What did you expect to see?
+
+
+
+3. What did you see instead?
+
+
+
+4. What version of BR and TiDB/TiKV/PD are you using?
+
+<!--
+br -V
+tidb-server -V
+tikv-server -V
+pd-server -V
+-->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,19 @@
+---
+name: "🚀 Feature Request"
+about: I have a suggestion
+labels: enhancement
+---
+
+## Feature Request
+
+### Describe your feature request related problem:
+<!-- A description of what the problem is. -->
+
+### Describe the feature you'd like:
+<!-- A description of what you want to happen. -->
+
+### Describe alternatives you've considered:
+<!-- A description of any alternative solutions or features you've considered. -->
+
+### Teachability, Documentation, Adoption, Migration Strategy:
+<!-- If you can, explain some scenarios how users might use this, or situations in which it would be helpful. Any API designs, mockups, or diagrams are also helpful. -->

.github/pull_request_template.md

@@ -0,0 +1,37 @@
+<!--
+Thank you for working on BR! Please read BR's [CONTRIBUTING](https://github.com/pingcap/br/blob/master/CONTRIBUTING.md) document **BEFORE** filing this PR.
+-->
+
+### What problem does this PR solve? <!--add issue link with summary if exists-->
+
+
+### What is changed and how it works?
+
+
+### Check List <!--REMOVE the items that are not applicable-->
+
+Tests <!-- At least one of them must be included. -->
+
+ - Unit test
+ - Integration test
+ - Manual test (add detailed scripts or steps below)
+ - No code
+
+Code changes
+
+ - Has exported function/method change
+ - Has exported variable/fields change
+ - Has interface methods change
+ - Has persistent data change
+
+Side effects
+
+ - Possible performance regression
+ - Increased code complexity
+ - Breaking backward compatibility
+
+Related changes
+
+ - Need to cherry-pick to the release branch
+ - Need to update the documentation
+ - Need to be included in the release note

CHANGELOG.md

@@ -0,0 +1,18 @@
+# BR (Backup and Restore) Change Log
+All notable changes to this project are documented in this file.
+See also,
+- [TiDB Changelog](https://github.com/pingcap/tidb/blob/master/CHANGELOG.md),
+- [TiKV Changelog](https://github.com/tikv/tikv/blob/master/CHANGELOG.md),
+- [PD Changelog](https://github.com/pingcap/pd/blob/master/CHANGELOG.md).
+
+## [3.1.0-beta.1] - 2020-01-10
+
+- Fix the inaccurate backup progress information [#127](https://github.com/pingcap/br/pull/127)
+- Improve the performance of splitting Regions [#122](https://github.com/pingcap/br/pull/122)
+- Add the backup and restore feature for partitioned tables [#137](https://github.com/pingcap/br/pull/137)
+- Add the feature of automatically scheduling PD schedulers [#123](https://github.com/pingcap/br/pull/123)
+- Fix the issue that data is overwritten after non `PKIsHandle` tables are restored [#139](https://github.com/pingcap/br/pull/139)
+
+## [3.1.0-beta] - 2019-12-20
+
+Initial release of the distributed backup and restore tool

Makefile

@@ -26,6 +26,8 @@ build_for_integration_test:
 		-o bin/br.test
 	# build key locker
 	GO111MODULE=on go build -race -o bin/locker tests/br_key_locked/*.go
+	# build gc
+	GO111MODULE=on go build -race -o bin/gc tests/br_z_gc_safepoint/*.go
 
 test:
 	GO111MODULE=on go test -race -tags leak ./...

cmd/backup.go

@@ -17,11 +17,12 @@ import (
 )
 
 const (
-	flagBackupTimeago     = "timeago"
-	flagBackupRateLimit   = "ratelimit"
-	flagBackupConcurrency = "concurrency"
-	flagBackupChecksum    = "checksum"
-	flagLastBackupTS      = "lastbackupts"
+	flagBackupTimeago       = "timeago"
+	flagBackupRateLimit     = "ratelimit"
+	flagBackupRateLimitUnit = "ratelimit-unit"
+	flagBackupConcurrency   = "concurrency"
+	flagBackupChecksum      = "checksum"
+	flagLastBackupTS        = "lastbackupts"
 )
 
 func defineBackupFlags(flagSet *pflag.FlagSet) {
@@ -36,6 +37,11 @@ func defineBackupFlags(flagSet *pflag.FlagSet) {
 		"Run checksum after backup")
 	flagSet.Uint64P(flagLastBackupTS, "", 0, "the last time backup ts")
 	_ = flagSet.MarkHidden(flagLastBackupTS)
+
+	// Test only flag.
+	flagSet.Uint64P(
+		flagBackupRateLimitUnit, "", utils.MB, "The unit of rate limit of the backup task")
+	_ = flagSet.MarkHidden(flagBackupRateLimitUnit)
 }
 
 func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error {
@@ -57,6 +63,11 @@ func runBackup(flagSet *pflag.FlagSet, cmdName, db, table string) error {
 	if err != nil {
 		return err
 	}
+	ratelimitUnit, err := flagSet.GetUint64(flagBackupRateLimitUnit)
+	if err != nil {
+		return err
+	}
+	ratelimit *= ratelimitUnit
 
 	concurrency, err := flagSet.GetUint32(flagBackupConcurrency)
 	if err != nil {

cmd/cmd.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/pingcap/br/pkg/conn"
 	"github.com/pingcap/br/pkg/storage"
+	"github.com/pingcap/br/pkg/utils"
 )
 
 var (
@@ -52,12 +53,18 @@ const (
 	FlagSlowLogFile = "slow-log-file"
 
 	flagDatabase = "db"
+	flagTable    = "table"
 
-	flagTable = "table"
+	flagVersion      = "version"
+	flagVersionShort = "V"
 )
 
 // AddFlags adds flags to the given cmd.
 func AddFlags(cmd *cobra.Command) {
+	cmd.Version = utils.BRInfo()
+	cmd.Flags().BoolP(flagVersion, flagVersionShort, false, "Display version information about BR")
+	cmd.SetVersionTemplate("{{printf \"%s\" .Version}}\n")
+
 	cmd.PersistentFlags().StringP(FlagPD, "u", "127.0.0.1:2379", "PD address")
 	cmd.PersistentFlags().String(FlagCA, "", "CA certificate path for TLS connection")
 	cmd.PersistentFlags().String(FlagCert, "", "Certificate path for TLS connection")

cmd/restore.go

@@ -111,6 +111,9 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error
 	case len(dbName) != 0 && len(tableName) == 0:
 		// database restore
 		db := client.GetDatabase(dbName)
+		if db == nil {
+			return errors.Errorf("database %s not found in backup", dbName)
+		}
 		err = client.CreateDatabase(db.Schema)
 		if err != nil {
 			return errors.Trace(err)
@@ -122,6 +125,9 @@ func runRestore(flagSet *flag.FlagSet, cmdName, dbName, tableName string) error
 	case len(dbName) != 0 && len(tableName) != 0:
 		// table restore
 		db := client.GetDatabase(dbName)
+		if db == nil {
+			return errors.Errorf("database %s not found in backup", dbName)
+		}
 		err = client.CreateDatabase(db.Schema)
 		if err != nil {
 			return errors.Trace(err)

main.go

@@ -47,7 +47,6 @@ func main() {
 	cmd.AddFlags(rootCmd)
 	cmd.SetDefaultContext(ctx)
 	rootCmd.AddCommand(
-		cmd.NewVersionCommand(),
 		cmd.NewValidateCommand(),
 		cmd.NewBackupCommand(),
 		cmd.NewRestoreCommand(),

pkg/backup/client.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/meta/autoid"
 	"github.com/pingcap/tidb/store/tikv"
+	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/ranger"
 	"go.uber.org/zap"
@@ -71,31 +72,31 @@ func (bc *Client) GetTS(ctx context.Context, timeAgo string) (uint64, error) {
 	if err != nil {
 		return 0, errors.Trace(err)
 	}
+	backupTS := oracle.ComposeTS(p, l)
 
 	if timeAgo != "" {
 		duration, err := time.ParseDuration(timeAgo)
 		if err != nil {
 			return 0, errors.Trace(err)
 		}
-		t := duration.Nanoseconds() / int64(time.Millisecond)
-		log.Info("backup time ago", zap.Int64("MillisecondsAgo", t))
-
-		// check backup time do not exceed GCSafePoint
-		safePoint, err := GetGCSafePoint(ctx, bc.mgr.GetPDClient())
-		if err != nil {
-			return 0, errors.Trace(err)
+		if duration <= 0 {
+			return 0, errors.New("negative timeago is not allowed")
 		}
-		if p-t < safePoint.Physical {
-			return 0, errors.New("given backup time exceed GCSafePoint")
+		log.Info("backup time ago", zap.Duration("timeago", duration))
+
+		backupTime := oracle.GetTimeFromTS(backupTS)
+		backupAgo := backupTime.Add(-duration)
+		if backupTS < oracle.ComposeTS(oracle.GetPhysical(backupAgo), l) {
+			return 0, errors.New("backup ts overflow please choose a smaller timeago")
 		}
-		p -= t
+		backupTS = oracle.ComposeTS(oracle.GetPhysical(backupAgo), l)
 	}
 
-	ts := utils.Timestamp{
-		Physical: p,
-		Logical:  l,
+	// check backup time do not exceed GCSafePoint
+	err = CheckGCSafepoint(ctx, bc.mgr.GetPDClient(), backupTS)
+	if err != nil {
+		return 0, errors.Trace(err)
 	}
-	backupTS := utils.EncodeTs(ts)
 	log.Info("backup encode timestamp", zap.Uint64("BackupTS", backupTS))
 	return backupTS, nil
 }
@@ -281,7 +282,7 @@ func (bc *Client) BackupRanges(
 	ranges []Range,
 	lastBackupTS uint64,
 	backupTS uint64,
-	rate uint64,
+	rateLimit uint64,
 	concurrency uint32,
 	updateCh chan<- struct{},
 ) error {
@@ -297,7 +298,7 @@ func (bc *Client) BackupRanges(
 	go func() {
 		for _, r := range ranges {
 			err := bc.backupRange(
-				ctx, r.StartKey, r.EndKey, lastBackupTS, backupTS, rate, concurrency, updateCh)
+				ctx, r.StartKey, r.EndKey, lastBackupTS, backupTS, rateLimit, concurrency, updateCh)
 			if err != nil {
 				errCh <- err
 				return
@@ -342,7 +343,7 @@ func (bc *Client) backupRange(
 	startKey, endKey []byte,
 	lastBackupTS uint64,
 	backupTS uint64,
-	rateMBs uint64,
+	rateLimit uint64,
 	concurrency uint32,
 	updateCh chan<- struct{},
 ) (err error) {
@@ -357,12 +358,10 @@ func (bc *Client) backupRange(
 			summary.CollectSuccessUnit(key, elapsed)
 		}
 	}()
-	// The unit of rate limit in protocol is bytes per second.
-	rateLimit := rateMBs * 1024 * 1024
 	log.Info("backup started",
 		zap.Binary("StartKey", startKey),
 		zap.Binary("EndKey", endKey),
-		zap.Uint64("RateLimit", rateMBs),
+		zap.Uint64("RateLimit", rateLimit),
 		zap.Uint32("Concurrency", concurrency))
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()

pkg/backup/client_test.go

@@ -10,11 +10,11 @@ import (
 	"github.com/pingcap/parser/model"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/store/mockstore/mocktikv"
+	"github.com/pingcap/tidb/store/tikv/oracle"
 	"github.com/pingcap/tidb/tablecodec"
 	"github.com/pingcap/tidb/util/codec"
 
 	"github.com/pingcap/br/pkg/conn"
-	"github.com/pingcap/br/pkg/utils"
 )
 
 type testBackup struct {
@@ -61,7 +61,7 @@ func (r *testBackup) TestGetTS(c *C) {
 	currentTs := time.Now().UnixNano() / int64(time.Millisecond)
 	ts, err := r.backupClient.GetTS(r.ctx, timeAgo)
 	c.Assert(err, IsNil)
-	pdTs := utils.DecodeTs(ts).Physical
+	pdTs := oracle.ExtractPhysical(ts)
 	duration := int(currentTs - pdTs)
 	c.Assert(duration, Greater, expectedDuration-deviation)
 	c.Assert(duration, Less, expectedDuration+deviation)
@@ -72,27 +72,30 @@ func (r *testBackup) TestGetTS(c *C) {
 	currentTs = time.Now().UnixNano() / int64(time.Millisecond)
 	ts, err = r.backupClient.GetTS(r.ctx, timeAgo)
 	c.Assert(err, IsNil)
-	pdTs = utils.DecodeTs(ts).Physical
+	pdTs = oracle.ExtractPhysical(ts)
 	duration = int(currentTs - pdTs)
 	c.Assert(duration, Greater, expectedDuration-deviation)
 	c.Assert(duration, Less, expectedDuration+deviation)
 
 	// timeago = "-1m"
 	timeAgo = "-1m"
-	expectedDuration = -60000
-	currentTs = time.Now().UnixNano() / int64(time.Millisecond)
-	ts, err = r.backupClient.GetTS(r.ctx, timeAgo)
-	c.Assert(err, IsNil)
-	pdTs = utils.DecodeTs(ts).Physical
-	duration = int(currentTs - pdTs)
-	c.Assert(duration, Greater, expectedDuration-deviation)
-	c.Assert(duration, Less, expectedDuration+deviation)
+	_, err = r.backupClient.GetTS(r.ctx, timeAgo)
+	c.Assert(err, ErrorMatches, "negative timeago is not allowed")
 
-	// timeago = "1000000h" exceed GCSafePoint
-	// because GCSafePoint in mockPDClient is 0
+	// timeago = "1000000h" overflows
 	timeAgo = "1000000h"
 	_, err = r.backupClient.GetTS(r.ctx, timeAgo)
-	c.Assert(err, ErrorMatches, "given backup time exceed GCSafePoint")
+	c.Assert(err, ErrorMatches, "backup ts overflow.*")
+
+	// timeago = "10h" exceed GCSafePoint
+	p, l, err := r.backupClient.mgr.GetPDClient().GetTS(r.ctx)
+	c.Assert(err, IsNil)
+	now := oracle.ComposeTS(p, l)
+	_, err = r.backupClient.mgr.GetPDClient().UpdateGCSafePoint(r.ctx, now)
+	c.Assert(err, IsNil)
+	timeAgo = "10h"
+	_, err = r.backupClient.GetTS(r.ctx, timeAgo)
+	c.Assert(err, ErrorMatches, "GC safepoint [0-9]+ exceed TS [0-9]+")
 }
 
 func (r *testBackup) TestBuildTableRange(c *C) {