release 3.1

go.mod

@@ -6,37 +6,30 @@ require (
 	cloud.google.com/go/storage v1.4.0
 	github.com/aws/aws-sdk-go v1.26.1
 	github.com/cheggaaa/pb/v3 v3.0.1
-	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
-	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect
 	github.com/fsouza/fake-gcs-server v1.15.0
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gogo/protobuf v1.3.1
-	github.com/golang/snappy v0.0.1 // indirect
 	github.com/google/btree v1.0.0
 	github.com/google/uuid v1.1.1
 	github.com/onsi/ginkgo v1.10.3 // indirect
 	github.com/onsi/gomega v1.7.1 // indirect
-	github.com/pingcap/check v0.0.0-20191107115940-caf2b9e6ccf4
+	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
 	github.com/pingcap/errors v0.11.4
-	github.com/pingcap/kvproto v0.0.0-20191212110315-d6a9d626988c
-	github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9
-	github.com/pingcap/parser v0.0.0-20191210060830-bdf23a7ade01
-	github.com/pingcap/pd v1.1.0-beta.0.20191212045800-234784c7a9c5
-	github.com/pingcap/tidb v1.1.0-beta.0.20191213040028-9009da737834
-	github.com/pingcap/tidb-tools v3.1.0-beta.0.20191223064326-e9c7a23a8dcb+incompatible
-	github.com/pingcap/tipb v0.0.0-20191209145133-44f75c9bef33
-	github.com/prometheus/client_golang v1.0.0
+	github.com/pingcap/kvproto v0.0.0-20191230111320-549d10f19d46
+	github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
+	github.com/pingcap/parser v0.0.0-20191205054626-288fe5207ce6
+	github.com/pingcap/pd v1.1.0-beta.0.20191115131715-6b7dc037010e
+	github.com/pingcap/tidb v1.1.0-beta.0.20191205065313-6083b21f986b
+	github.com/pingcap/tidb-tools v3.1.0-beta.0.20191230034204-f90021ce2de1+incompatible
+	github.com/pingcap/tipb v0.0.0-20191126033718-169898888b24
+	github.com/prometheus/client_golang v1.1.0
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.3
 	go.opencensus.io v0.22.2 // indirect
-	go.uber.org/atomic v1.5.1 // indirect
-	go.uber.org/zap v1.13.0
-	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
+	go.uber.org/zap v1.10.0
 	golang.org/x/net v0.0.0-20191011234655-491137f69257 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/tools v0.0.0-20191213032237-7093a17b0467 // indirect
 	google.golang.org/api v0.14.0
-	google.golang.org/grpc v1.25.1
+	google.golang.org/grpc v1.24.0
 )

pkg/backup/client.go

@@ -549,7 +549,6 @@ func (bc *Client) fineGrainedBackup(
 
 func onBackupResponse(
 	bo *tikv.Backoffer,
-	backupTS uint64,
 	lockResolver *tikv.LockResolver,
 	resp *backup.BackupResponse,
 ) (*backup.BackupResponse, int, error) {
@@ -563,8 +562,8 @@ func onBackupResponse(
 		if lockErr := v.KvError.Locked; lockErr != nil {
 			// Try to resolve lock.
 			log.Warn("backup occur kv error", zap.Reflect("error", v))
-			msBeforeExpired, _, err1 := lockResolver.ResolveLocks(
-				bo, backupTS, []*tikv.Lock{tikv.NewLock(lockErr)})
+			msBeforeExpired, err1 := lockResolver.ResolveLocks(
+				bo, []*tikv.Lock{tikv.NewLock(lockErr)})
 			if err1 != nil {
 				return nil, 0, errors.Trace(err1)
 			}
@@ -646,7 +645,7 @@ func (bc *Client) handleFineGrained(
 		// Handle responses with the same backoffer.
 		func(resp *backup.BackupResponse) error {
 			response, backoffMs, err1 :=
-				onBackupResponse(bo, backupTS, lockResolver, resp)
+				onBackupResponse(bo, lockResolver, resp)
 			if err1 != nil {
 				return err1
 			}

pkg/checksum/executor.go

@@ -147,6 +147,7 @@ func buildTableRequest(
 	}
 
 	checksum := &tipb.ChecksumRequest{
+		StartTs:   startTS,
 		ScanOn:    tipb.ChecksumScanOn_Table,
 		Algorithm: tipb.ChecksumAlgorithm_Crc64_Xor,
 		Rule:      rule,
@@ -158,7 +159,6 @@ func buildTableRequest(
 	// Use low priority to reducing impact to other requests.
 	builder.Request.Priority = kv.PriorityLow
 	return builder.SetTableRanges(tableID, ranges, nil).
-		SetStartTS(startTS).
 		SetChecksumRequest(checksum).
 		SetConcurrency(variable.DefDistSQLScanConcurrency).
 		Build()
@@ -179,6 +179,7 @@ func buildIndexRequest(
 		}
 	}
 	checksum := &tipb.ChecksumRequest{
+		StartTs:   startTS,
 		ScanOn:    tipb.ChecksumScanOn_Index,
 		Algorithm: tipb.ChecksumAlgorithm_Crc64_Xor,
 		Rule:      rule,
@@ -190,7 +191,6 @@ func buildIndexRequest(
 	// Use low priority to reducing impact to other requests.
 	builder.Request.Priority = kv.PriorityLow
 	return builder.SetIndexRanges(nil, tableID, indexInfo.ID, ranges).
-		SetStartTS(startTS).
 		SetChecksumRequest(checksum).
 		SetConcurrency(variable.DefDistSQLScanConcurrency).
 		Build()

pkg/conn/conn.go

@@ -25,7 +25,6 @@ import (
 	"github.com/pingcap/tidb/util/codec"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/keepalive"
 )
 
@@ -219,13 +218,11 @@ func (mgr *Mgr) getGrpcConnLocked(ctx context.Context, storeID uint64) (*grpc.Cl
 	ctx, cancel := context.WithTimeout(ctx, dialTimeout)
 	keepAlive := 10
 	keepAliveTimeout := 3
-	bfConf := backoff.DefaultConfig
-	bfConf.MaxDelay = time.Second * 3
 	conn, err := grpc.DialContext(
 		ctx,
 		store.GetAddress(),
 		opt,
-		grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
+		grpc.WithBackoffMaxDelay(time.Second*3),
 		grpc.WithKeepaliveParams(keepalive.ClientParameters{
 			Time:                time.Duration(keepAlive) * time.Second,
 			Timeout:             time.Duration(keepAliveTimeout) * time.Second,

pkg/conn/conn_test.go

@@ -22,6 +22,10 @@ func TestT(t *testing.T) {
 	TestingT(t)
 }
 
+func TestClient(t *testing.T) {
+	TestingT(t)
+}
+
 var _ = Suite(&testClientSuite{})
 
 type testClientSuite struct {

pkg/restore/client.go

@@ -18,7 +18,6 @@ import (
 	"github.com/pingcap/tidb/kv"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/backoff"
 	"google.golang.org/grpc/keepalive"
 
 	"github.com/pingcap/br/pkg/checksum"
@@ -389,8 +388,6 @@ func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMo
 	if err != nil {
 		return errors.Trace(err)
 	}
-	bfConf := backoff.DefaultConfig
-	bfConf.MaxDelay = time.Second * 3
 	for _, store := range stores {
 		opt := grpc.WithInsecure()
 		gctx, cancel := context.WithTimeout(ctx, time.Second*5)
@@ -400,7 +397,7 @@ func (rc *Client) switchTiKVMode(ctx context.Context, mode import_sstpb.SwitchMo
 			gctx,
 			store.GetAddress(),
 			opt,
-			grpc.WithConnectParams(grpc.ConnectParams{Backoff: bfConf}),
+			grpc.WithBackoffMaxDelay(time.Second*3),
 			grpc.WithKeepaliveParams(keepalive.ClientParameters{
 				Time:                time.Duration(keepAlive) * time.Second,
 				Timeout:             time.Duration(keepAliveTimeout) * time.Second,

pkg/restore/db.go

@@ -92,6 +92,19 @@ func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error {
 			zap.Error(err))
 		return errors.Trace(err)
 	}
+	alterAutoIncIDSQL := fmt.Sprintf(
+		"alter table %s auto_increment = %d",
+		escapeTableName(schema.Name),
+		schema.AutoIncID)
+	_, err = db.se.Execute(ctx, alterAutoIncIDSQL)
+	if err != nil {
+		log.Error("alter AutoIncID failed",
+			zap.String("SQL", alterAutoIncIDSQL),
+			zap.Stringer("db", table.Db.Name),
+			zap.Stringer("table", table.Schema.Name),
+			zap.Error(err))
+		return errors.Trace(err)
+	}
 	return nil
 }
 

pkg/restore/db_test.go

@@ -41,23 +41,23 @@ func (s *testRestoreSchemaSuite) TestRestoreAutoIncID(c *C) {
 	tk := testkit.NewTestKit(c, s.mock.Storage)
 	tk.MustExec("use test")
 	tk.MustExec("set @@sql_mode=''")
-	tk.MustExec("drop table if exists t;")
+	tk.MustExec("drop table if exists `\"t\"`;")
 	// Test SQL Mode
-	tk.MustExec("create table t (" +
-		"a int not null auto_increment," +
+	tk.MustExec("create table `\"t\"` (" +
+		"a int not null," +
 		"time timestamp not null default '0000-00-00 00:00:00'," +
 		"primary key (a));",
 	)
-	tk.MustExec("insert into t values (10, '0000-00-00 00:00:00');")
+	tk.MustExec("insert into `\"t\"` values (10, '0000-00-00 00:00:00');")
 	// Query the current AutoIncID
-	autoIncID, err := strconv.ParseUint(tk.MustQuery("admin show t next_row_id").Rows()[0][3].(string), 10, 64)
+	autoIncID, err := strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
 	c.Assert(err, IsNil, Commentf("Error query auto inc id: %s", err))
 	// Get schemas of db and table
 	info, err := s.mock.Domain.GetSnapshotInfoSchema(math.MaxUint64)
 	c.Assert(err, IsNil, Commentf("Error get snapshot info schema: %s", err))
 	dbInfo, exists := info.SchemaByName(model.NewCIStr("test"))
 	c.Assert(exists, IsTrue, Commentf("Error get db info"))
-	tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	tableInfo, err := info.TableByName(model.NewCIStr("test"), model.NewCIStr("\"t\""))
 	c.Assert(err, IsNil, Commentf("Error get table info: %s", err))
 	table := utils.Table{
 		Schema: tableInfo.Meta(),
@@ -88,7 +88,7 @@ func (s *testRestoreSchemaSuite) TestRestoreAutoIncID(c *C) {
 	c.Assert(err, IsNil, Commentf("Error create table: %s %s", err, s.mock.DSN))
 	tk.MustExec("use test")
 	// Check if AutoIncID is altered successfully
-	autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show t next_row_id").Rows()[0][3].(string), 10, 64)
+	autoIncID, err = strconv.ParseUint(tk.MustQuery("admin show `\"t\"` next_row_id").Rows()[0][3].(string), 10, 64)
 	c.Assert(err, IsNil, Commentf("Error query auto inc id: %s", err))
 	c.Assert(autoIncID, Equals, uint64(globalAutoID+100))
 }

pkg/restore/import.go

@@ -11,8 +11,8 @@ import (
 	"github.com/pingcap/kvproto/pkg/import_sstpb"
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
 	"github.com/pingcap/log"
-	"github.com/pingcap/pd/pkg/codec"
 	restore_util "github.com/pingcap/tidb-tools/pkg/restore-util"
+	"github.com/pingcap/tidb/util/codec"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
@@ -264,7 +264,7 @@ func (importer *FileImporter) downloadSST(
 		return nil, true, errors.Trace(err)
 	}
 	// Assume one region reflects to one rewrite rule
-	_, key, err := codec.DecodeBytes(regionInfo.Region.GetStartKey())
+	_, key, err := codec.DecodeBytes(regionInfo.Region.GetStartKey(), []byte{})
 	if err != nil {
 		return nil, true, err
 	}

pkg/restore/util.go

@@ -368,3 +368,12 @@ func encodeKeyPrefix(key []byte) []byte {
 	encodedPrefix = append(encodedPrefix, codec.EncodeBytes([]byte{}, key[:len(key)-ungroupedLen])...)
 	return append(encodedPrefix[:len(encodedPrefix)-9], key[len(key)-ungroupedLen:]...)
 }
+
+// escape the identifier for pretty-printing.
+// For instance, the identifier "foo `bar`" will become "`foo ``bar```".
+// The sqlMode controls whether to escape with backquotes (`) or double quotes
+// (`"`) depending on whether mysql.ModeANSIQuotes is enabled.
+func escapeTableName(cis model.CIStr) string {
+	quote := "`"
+	return quote + strings.Replace(cis.O, quote, quote+quote, -1) + quote
+}

tests/br_insert_after_restore/run.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+#
+# Copyright 2019 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+DB="$TEST_NAME"
+TABLE="usertable"
+ROW_COUNT=10
+PATH="tests/$TEST_NAME:bin:$PATH"
+
+insertRecords() {
+    for i in $(seq $1); do
+        run_sql "INSERT INTO $DB.$TABLE VALUES ('$i');"
+    done
+}
+
+createTable() {
+    run_sql "CREATE TABLE IF NOT EXISTS $DB.$TABLE (c1 CHAR(255));"
+}
+
+echo "load data..."
+echo "create database"
+run_sql "CREATE DATABASE IF NOT EXISTS $DB;"
+echo "create table"
+createTable
+echo "insert records"
+insertRecords $ROW_COUNT
+
+row_count_ori=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}')
+
+# backup full
+echo "backup start..."
+run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB" --ratelimit 5 --concurrency 4
+
+run_sql "DROP DATABASE $DB;"
+
+# restore full
+echo "restore start..."
+run_br restore full -s "local://$TEST_DIR/$DB" --pd $PD_ADDR
+
+row_count_new=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}')
+
+fail=false
+if [ "${row_count_ori}" != "${row_count_new}" ];then
+    fail=true
+    echo "TEST: [$TEST_NAME] fail on database $DB"
+fi
+echo "database $DB [original] row count: ${row_count_ori}, [after br] row count: ${row_count_new}"
+
+if $fail; then
+    echo "TEST: [$TEST_NAME] failed!"
+    exit 1
+fi
+
+# insert records
+insertRecords $ROW_COUNT
+row_count_insert=$(run_sql "SELECT COUNT(*) FROM $DB.$TABLE;" | awk '/COUNT/{print $2}')
+fail=false
+if [ "${row_count_insert}" != "$(expr $row_count_new \* 2)" ];then
+    fail=true
+    echo "TEST: [$TEST_NAME] fail on inserting records to database $DB after restore: ${row_count_insert}"
+fi
+
+if $fail; then
+    echo "TEST: [$TEST_NAME] failed!"
+    exit 1
+else
+    echo "TEST: [$TEST_NAME] successed!"
+fi
+
+run_sql "DROP DATABASE $DB;"

tests/br_key_locked/locker.go

@@ -260,13 +260,15 @@ func (c *Locker) lockBatch(ctx context.Context, keys [][]byte, primary []byte) (
 			return 0, nil
 		}
 
-		prewrite := &kvrpcpb.PrewriteRequest{
-			Mutations:    mutations,
-			PrimaryLock:  primary,
-			StartVersion: startTs,
-			LockTtl:      uint64(c.lockTTL.Milliseconds()),
+		req := &tikvrpc.Request{
+			Type: tikvrpc.CmdPrewrite,
+			Prewrite: &kvrpcpb.PrewriteRequest{
+				Mutations:    mutations,
+				PrimaryLock:  primary,
+				StartVersion: startTs,
+				LockTtl:      uint64(c.lockTTL.Milliseconds()),
+			},
 		}
-		req := tikvrpc.NewRequest(tikvrpc.CmdPrewrite, prewrite)
 
 		// Send the requests
 		resp, err := c.kv.SendReq(bo, req, loc.Region, time.Second*20)
@@ -288,7 +290,7 @@ func (c *Locker) lockBatch(ctx context.Context, keys [][]byte, primary []byte) (
 			continue
 		}
 
-		prewriteResp := resp.Resp
+		prewriteResp := resp.Prewrite
 		if prewriteResp == nil {
 			return 0, errors.Errorf("response body missing")
 		}