From fa7fbf1e2b583d4fb3fd886ec157e142ea709cc5 Mon Sep 17 00:00:00 2001 From: kennytm Date: Wed, 17 Mar 2021 16:18:58 +0800 Subject: [PATCH] cherry pick #877 to release-5.0 Signed-off-by: ti-srebot --- Makefile | 6 +- cmd/tidb-lightning-ctl/main.go | 41 +-- pkg/lightning/backend/backend.go | 70 +---- pkg/lightning/backend/backend_test.go | 14 +- pkg/lightning/backend/checkreq_test.go | 167 ------------ .../backend/{ => importer}/importer.go | 157 +++-------- .../backend/{ => importer}/importer_test.go | 77 ++++-- pkg/lightning/backend/{ => kv}/allocator.go | 4 +- pkg/lightning/backend/{ => kv}/session.go | 9 +- .../backend/{ => kv}/session_test.go | 2 +- pkg/lightning/backend/{ => kv}/sql2kv.go | 30 ++- pkg/lightning/backend/{ => kv}/sql2kv_test.go | 6 +- pkg/lightning/backend/kv/types.go | 50 ++++ pkg/lightning/backend/{ => local}/local.go | 132 +++++---- .../backend/{ => local}/local_test.go | 38 +-- .../backend/{ => local}/local_unix.go | 2 +- .../backend/{ => local}/local_windows.go | 2 +- .../backend/{ => local}/localhelper.go | 2 +- .../backend/{ => local}/localhelper_test.go | 2 +- pkg/lightning/backend/{ => tidb}/tidb.go | 55 ++-- pkg/lightning/backend/{ => tidb}/tidb_test.go | 29 +- pkg/lightning/backend/tikv_test.go | 148 ----------- pkg/lightning/lightning.go | 4 +- pkg/lightning/lightning_test.go | 12 +- pkg/lightning/restore/restore.go | 57 ++-- pkg/lightning/restore/restore_test.go | 23 +- pkg/lightning/{backend => tikv}/tikv.go | 42 ++- pkg/lightning/tikv/tikv_test.go | 251 ++++++++++++++++++ pkg/mock/backend.go | 245 ++++------------- pkg/mock/dummy.go | 5 - pkg/mock/glue.go | 2 - pkg/mock/glue_checkpoint.go | 2 - pkg/mock/importer.go | 2 - pkg/mock/kv.go | 152 +++++++++++ pkg/mock/mock_cluster.go | 2 - pkg/mock/mock_cluster_test.go | 2 - pkg/mock/s3iface.go | 2 - pkg/mock/storage.go | 2 - pkg/version/version.go | 18 +- pkg/version/version_test.go | 14 + tests/README.md | 4 +- tests/br_log_restore/run.sh | 4 +- tests/lightning_local_backend/run.sh | 4 +- tests/lightning_tidb_duplicate_data/run.sh | 2 +- 44 files changed, 961 insertions(+), 933 deletions(-) delete mode 100644 pkg/lightning/backend/checkreq_test.go rename pkg/lightning/backend/{ => importer}/importer.go (63%) rename pkg/lightning/backend/{ => importer}/importer_test.go (69%) rename pkg/lightning/backend/{ => kv}/allocator.go (96%) rename pkg/lightning/backend/{ => kv}/session.go (96%) rename pkg/lightning/backend/{ => kv}/session_test.go (98%) rename pkg/lightning/backend/{ => kv}/sql2kv.go (93%) rename pkg/lightning/backend/{ => kv}/sql2kv_test.go (99%) create mode 100644 pkg/lightning/backend/kv/types.go rename pkg/lightning/backend/{ => local}/local.go (94%) rename pkg/lightning/backend/{ => local}/local_test.go (94%) rename pkg/lightning/backend/{ => local}/local_unix.go (99%) rename pkg/lightning/backend/{ => local}/local_windows.go (98%) rename pkg/lightning/backend/{ => local}/localhelper.go (99%) rename pkg/lightning/backend/{ => local}/localhelper_test.go (99%) rename pkg/lightning/backend/{ => tidb}/tidb.go (90%) rename pkg/lightning/backend/{ => tidb}/tidb_test.go (94%) delete mode 100644 pkg/lightning/backend/tikv_test.go rename pkg/lightning/{backend => tikv}/tikv.go (82%) create mode 100644 pkg/lightning/tikv/tikv_test.go delete mode 100644 pkg/mock/dummy.go create mode 100644 pkg/mock/kv.go diff --git a/Makefile b/Makefile index 70f1b296b..67c93b36a 100644 --- a/Makefile +++ b/Makefile @@ -112,14 +112,14 @@ test: export ARGS=$$($(PACKAGES)) test: $(PREPARE_MOD) @make failpoint-enable - $(GOTEST) $(RACEFLAG) -tags br_test,leak $(ARGS) || ( make failpoint-disable && exit 1 ) + $(GOTEST) $(RACEFLAG) -tags leak $(ARGS) || ( make failpoint-disable && exit 1 ) @make failpoint-disable testcover: tools mkdir -p "$(TEST_DIR)" $(PREPARE_MOD) @make failpoint-enable - $(GOTEST) -tags br_test -cover -covermode=count -coverprofile="$(TEST_DIR)/cov.unit.out" \ + $(GOTEST) -cover -covermode=count -coverprofile="$(TEST_DIR)/cov.unit.out" \ $$($(COVERED_PACKAGES)) || ( make failpoint-disable && exit 1 ) @make failpoint-disable @@ -186,7 +186,7 @@ static: prepare tools @# exhaustivestruct - Protobuf structs have hidden fields, like "XXX_NoUnkeyedLiteral" @# exhaustive - no need to check exhaustiveness of enum switch statements @# gosec - too many false positive - CGO_ENABLED=0 tools/bin/golangci-lint run --enable-all --build-tags br_test --deadline 120s \ + CGO_ENABLED=0 tools/bin/golangci-lint run --enable-all --deadline 120s \ --disable gochecknoglobals \ --disable goimports \ --disable gofmt \ diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 54e04a312..47d8bc8ba 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -26,11 +26,14 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/import_sstpb" - kv "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/importer" + "github.com/pingcap/br/pkg/lightning/backend/local" "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/restore" + "github.com/pingcap/br/pkg/lightning/tikv" ) func main() { @@ -133,12 +136,12 @@ func run() error { } func compactCluster(ctx context.Context, cfg *config.Config, tls *common.TLS) error { - return kv.ForAllStores( + return tikv.ForAllStores( ctx, tls.WithHost(cfg.TiDB.PdAddr), - kv.StoreStateDisconnected, - func(c context.Context, store *kv.Store) error { - return kv.Compact(c, tls, store.Address, restore.FullLevelCompact) + tikv.StoreStateDisconnected, + func(c context.Context, store *tikv.Store) error { + return tikv.Compact(c, tls, store.Address, restore.FullLevelCompact) }, ) } @@ -154,23 +157,23 @@ func switchMode(ctx context.Context, cfg *config.Config, tls *common.TLS, mode s return errors.Errorf("invalid mode %s, must use %s or %s", mode, config.ImportMode, config.NormalMode) } - return kv.ForAllStores( + return tikv.ForAllStores( ctx, tls.WithHost(cfg.TiDB.PdAddr), - kv.StoreStateDisconnected, - func(c context.Context, store *kv.Store) error { - return kv.SwitchMode(c, tls, store.Address, m) + tikv.StoreStateDisconnected, + func(c context.Context, store *tikv.Store) error { + return tikv.SwitchMode(c, tls, store.Address, m) }, ) } func fetchMode(ctx context.Context, cfg *config.Config, tls *common.TLS) error { - return kv.ForAllStores( + return tikv.ForAllStores( ctx, tls.WithHost(cfg.TiDB.PdAddr), - kv.StoreStateDisconnected, - func(c context.Context, store *kv.Store) error { - mode, err := kv.FetchMode(c, tls, store.Address) + tikv.StoreStateDisconnected, + func(c context.Context, store *tikv.Store) error { + mode, err := tikv.FetchMode(c, tls, store.Address) if err != nil { fmt.Fprintf(os.Stderr, "%-30s | Error: %v\n", store.Address, err) } else { @@ -231,7 +234,7 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common } if cfg.TikvImporter.Backend == "importer" { - importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + importer, err := importer.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } @@ -259,8 +262,8 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common for _, table := range targetTables { for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ { fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID) - _, eID := kv.MakeUUID(table.TableName, engineID) - file := kv.LocalFile{Uuid: eID} + _, eID := backend.MakeUUID(table.TableName, engineID) + file := local.File{Uuid: eID} err := file.Cleanup(cfg.TikvImporter.SortedKVDir) if err != nil { fmt.Fprintln(os.Stderr, "* Encountered error while cleanup engine:", err) @@ -357,7 +360,7 @@ func getLocalStoringTables(ctx context.Context, cfg *config.Config) (err2 error) return nil } -func unsafeCloseEngine(ctx context.Context, importer kv.Backend, engine string) (*kv.ClosedEngine, error) { +func unsafeCloseEngine(ctx context.Context, importer backend.Backend, engine string) (*backend.ClosedEngine, error) { if index := strings.LastIndexByte(engine, ':'); index >= 0 { tableName := engine[:index] engineID, err := strconv.Atoi(engine[index+1:]) @@ -378,7 +381,7 @@ func unsafeCloseEngine(ctx context.Context, importer kv.Backend, engine string) } func importEngine(ctx context.Context, cfg *config.Config, tls *common.TLS, engine string) error { - importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + importer, err := importer.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } @@ -392,7 +395,7 @@ func importEngine(ctx context.Context, cfg *config.Config, tls *common.TLS, engi } func cleanupEngine(ctx context.Context, cfg *config.Config, tls *common.TLS, engine string) error { - importer, err := kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + importer, err := importer.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return errors.Trace(err) } diff --git a/pkg/lightning/backend/backend.go b/pkg/lightning/backend/backend.go index dad35c526..b75d5ccf6 100644 --- a/pkg/lightning/backend/backend.go +++ b/pkg/lightning/backend/backend.go @@ -24,17 +24,16 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/table" - "github.com/pingcap/tidb/types" "go.uber.org/zap" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/log" "github.com/pingcap/br/pkg/lightning/metric" - "github.com/pingcap/br/pkg/lightning/verification" ) const ( - maxRetryTimes = 3 // tikv-importer has done retry internally. so we don't retry many times. + importMaxRetryTimes = 3 // tikv-importer has done retry internally. so we don't retry many times. ) /* @@ -102,7 +101,7 @@ type AbstractBackend interface { Close() // MakeEmptyRows creates an empty collection of encoded rows. - MakeEmptyRows() Rows + MakeEmptyRows() kv.Rows // RetryImportDelay returns the duration to sleep when retrying an import RetryImportDelay() time.Duration @@ -112,7 +111,7 @@ type AbstractBackend interface { ShouldPostProcess() bool // NewEncoder creates an encoder of a TiDB table. - NewEncoder(tbl table.Table, options *SessionOptions) (Encoder, error) + NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) OpenEngine(ctx context.Context, engineUUID uuid.UUID) error @@ -165,15 +164,6 @@ type AbstractBackend interface { LocalWriter(ctx context.Context, engineUUID uuid.UUID) (EngineWriter, error) } -func fetchRemoteTableModelsFromTLS(ctx context.Context, tls *common.TLS, schema string) ([]*model.TableInfo, error) { - var tables []*model.TableInfo - err := tls.GetJSON(ctx, "/schema/"+schema, &tables) - if err != nil { - return nil, errors.Annotatef(err, "cannot read schema '%s' from remote", schema) - } - return tables, nil -} - // Backend is the delivery target for Lightning type Backend struct { abstract AbstractBackend @@ -221,11 +211,11 @@ func (be Backend) Close() { be.abstract.Close() } -func (be Backend) MakeEmptyRows() Rows { +func (be Backend) MakeEmptyRows() kv.Rows { return be.abstract.MakeEmptyRows() } -func (be Backend) NewEncoder(tbl table.Table, options *SessionOptions) (Encoder, error) { +func (be Backend) NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) { return be.abstract.NewEncoder(tbl, options) } @@ -353,7 +343,7 @@ func (engine *OpenedEngine) LocalWriter(ctx context.Context) (*LocalEngineWriter } // WriteRows writes a collection of encoded rows into the engine. -func (w *LocalEngineWriter) WriteRows(ctx context.Context, columnNames []string, rows Rows) error { +func (w *LocalEngineWriter) WriteRows(ctx context.Context, columnNames []string, rows kv.Rows) error { return w.writer.AppendRows(ctx, w.tableName, columnNames, w.ts, rows) } @@ -398,7 +388,7 @@ func (en engine) unsafeClose(ctx context.Context) (*ClosedEngine, error) { func (engine *ClosedEngine) Import(ctx context.Context) error { var err error - for i := 0; i < maxRetryTimes; i++ { + for i := 0; i < importMaxRetryTimes; i++ { task := engine.logger.With(zap.Int("retryCnt", i)).Begin(zap.InfoLevel, "import") err = engine.backend.ImportEngine(ctx, engine.uuid) if !common.IsRetryableError(err) { @@ -409,7 +399,7 @@ func (engine *ClosedEngine) Import(ctx context.Context) error { time.Sleep(engine.backend.RetryImportDelay()) } - return errors.Annotatef(err, "[%s] import reach max retry %d and still failed", engine.uuid, maxRetryTimes) + return errors.Annotatef(err, "[%s] import reach max retry %d and still failed", engine.uuid, importMaxRetryTimes) } // Cleanup deletes the intermediate data from target. @@ -424,53 +414,13 @@ func (engine *ClosedEngine) Logger() log.Logger { return engine.logger } -// Encoder encodes a row of SQL values into some opaque type which can be -// consumed by OpenEngine.WriteEncoded. -type Encoder interface { - // Close the encoder. - Close() - - // Encode encodes a row of SQL values into a backend-friendly format. - Encode( - logger log.Logger, - row []types.Datum, - rowID int64, - columnPermutation []int, - ) (Row, error) -} - -// Row represents a single encoded row. -type Row interface { - // ClassifyAndAppend separates the data-like and index-like parts of the - // encoded row, and appends these parts into the existing buffers and - // checksums. - ClassifyAndAppend( - data *Rows, - dataChecksum *verification.KVChecksum, - indices *Rows, - indexChecksum *verification.KVChecksum, - ) -} - -// Rows represents a collection of encoded rows. -type Rows interface { - // SplitIntoChunks splits the rows into multiple consecutive parts, each - // part having total byte size less than `splitSize`. The meaning of "byte - // size" should be consistent with the value used in `Row.ClassifyAndAppend`. - SplitIntoChunks(splitSize int) []Rows - - // Clear returns a new collection with empty content. It may share the - // capacity with the current instance. The typical usage is `x = x.Clear()`. - Clear() Rows -} - type EngineWriter interface { AppendRows( ctx context.Context, tableName string, columnNames []string, commitTS uint64, - rows Rows, + rows kv.Rows, ) error Close() error } diff --git a/pkg/lightning/backend/backend_test.go b/pkg/lightning/backend/backend_test.go index 92ddf1535..5094420ca 100644 --- a/pkg/lightning/backend/backend_test.go +++ b/pkg/lightning/backend/backend_test.go @@ -2,6 +2,7 @@ package backend_test import ( "context" + "testing" "time" "github.com/golang/mock/gomock" @@ -11,26 +12,31 @@ import ( "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/store/tikv/oracle" - kv "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/mock" ) type backendSuite struct { controller *gomock.Controller mockBackend *mock.MockBackend - backend kv.Backend + backend backend.Backend ts uint64 } var _ = Suite(&backendSuite{}) +func Test(t *testing.T) { + TestingT(t) +} + // FIXME: Cannot use the real SetUpTest/TearDownTest to set up the mock // otherwise the mock error will be ignored. func (s *backendSuite) setUpTest(c *C) { s.controller = gomock.NewController(c) s.mockBackend = mock.NewMockBackend(s.controller) - s.backend = kv.MakeBackend(s.mockBackend) + s.backend = backend.MakeBackend(s.mockBackend) s.ts = oracle.ComposeTS(time.Now().Unix()*1000, 0) } @@ -334,7 +340,7 @@ func (s *backendSuite) TestCheckDiskQuota(c *C) { uuid7 := uuid.MustParse("77777777-7777-7777-7777-777777777777") uuid9 := uuid.MustParse("99999999-9999-9999-9999-999999999999") - fileSizes := []kv.EngineFileSize{ + fileSizes := []backend.EngineFileSize{ { UUID: uuid1, DiskSize: 1000, diff --git a/pkg/lightning/backend/checkreq_test.go b/pkg/lightning/backend/checkreq_test.go deleted file mode 100644 index c2316ccee..000000000 --- a/pkg/lightning/backend/checkreq_test.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package backend - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - - "github.com/coreos/go-semver/semver" - . "github.com/pingcap/check" - - "github.com/pingcap/br/pkg/lightning/common" -) - -var _ = Suite(&checkReqSuite{}) - -type checkReqSuite struct{} - -func (s *checkReqSuite) TestCheckVersion(c *C) { - err := checkVersion("TiNB", *semver.New("2.3.5"), *semver.New("2.1.0"), *semver.New("3.0.0")) - c.Assert(err, IsNil) - - err = checkVersion("TiNB", *semver.New("2.1.0"), *semver.New("2.3.5"), *semver.New("3.0.0")) - c.Assert(err, ErrorMatches, "TiNB version too old.*") - - err = checkVersion("TiNB", *semver.New("3.1.0"), *semver.New("2.3.5"), *semver.New("3.0.0")) - c.Assert(err, ErrorMatches, "TiNB version too new.*") - - err = checkVersion("TiNB", *semver.New("3.0.0-beta"), *semver.New("2.3.5"), *semver.New("3.0.0")) - c.Assert(err, ErrorMatches, "TiNB version too new.*") -} - -func (s *checkReqSuite) TestCheckTiDBVersion(c *C) { - var version string - ctx := context.Background() - - mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - c.Assert(req.URL.Path, Equals, "/status") - w.WriteHeader(http.StatusOK) - err := json.NewEncoder(w).Encode(map[string]interface{}{ - "version": version, - }) - c.Assert(err, IsNil) - })) - - tls := common.NewTLSFromMockServer(mockServer) - - version = "5.7.25-TiDB-v4.0.0" - c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), IsNil) - - version = "5.7.25-TiDB-v9999.0.0" - c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too new.*") - - version = "5.7.25-TiDB-v6.0.0" - c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too new.*") - - version = "5.7.25-TiDB-v6.0.0-beta" - c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too new.*") - - version = "5.7.25-TiDB-v1.0.0" - c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too old.*") -} - -func (s *checkReqSuite) TestCheckPDVersion(c *C) { - var version string - ctx := context.Background() - - mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - c.Assert(req.URL.Path, Equals, "/pd/api/v1/version") - w.WriteHeader(http.StatusOK) - _, err := w.Write([]byte(version)) - c.Assert(err, IsNil) - })) - mockURL, err := url.Parse(mockServer.URL) - c.Assert(err, IsNil) - - tls := common.NewTLSFromMockServer(mockServer) - - version = `{ - "version": "v4.0.0-rc.2-451-g760fb650" -}` - c.Assert(checkPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), IsNil) - - version = `{ - "version": "v4.0.0" -}` - c.Assert(checkPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), IsNil) - - version = `{ - "version": "v9999.0.0" -}` - c.Assert(checkPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") - - version = `{ - "version": "v6.0.0" -}` - c.Assert(checkPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") - - version = `{ - "version": "v6.0.0-beta" -}` - c.Assert(checkPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") - - version = `{ - "version": "v1.0.0" -}` - c.Assert(checkPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too old.*") -} - -func (s *checkReqSuite) TestCheckTiKVVersion(c *C) { - var versions []string - ctx := context.Background() - - mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - c.Assert(req.URL.Path, Equals, "/pd/api/v1/stores") - w.WriteHeader(http.StatusOK) - - stores := make([]map[string]interface{}, 0, len(versions)) - for i, v := range versions { - stores = append(stores, map[string]interface{}{ - "store": map[string]interface{}{ - "address": fmt.Sprintf("tikv%d.test:20160", i), - "version": v, - }, - }) - } - err := json.NewEncoder(w).Encode(map[string]interface{}{ - "count": len(versions), - "stores": stores, - }) - c.Assert(err, IsNil) - })) - mockURL, err := url.Parse(mockServer.URL) - c.Assert(err, IsNil) - - tls := common.NewTLSFromMockServer(mockServer) - - versions = []string{"4.1.0", "v4.1.0-alpha-9-ga27a7dd"} - c.Assert(checkTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), IsNil) - - versions = []string{"9999.0.0", "4.0.0"} - c.Assert(checkTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) - - versions = []string{"4.0.0", "1.0.0"} - c.Assert(checkTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv1\.test:20160\) version too old.*`) - - versions = []string{"6.0.0"} - c.Assert(checkTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) - - versions = []string{"6.0.0-beta"} - c.Assert(checkTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) -} diff --git a/pkg/lightning/backend/importer.go b/pkg/lightning/backend/importer/importer.go similarity index 63% rename from pkg/lightning/backend/importer.go rename to pkg/lightning/backend/importer/importer.go index d63cc3ef1..4f210812a 100644 --- a/pkg/lightning/backend/importer.go +++ b/pkg/lightning/backend/importer/importer.go @@ -11,11 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package importer import ( "context" - "fmt" "strings" "sync" "time" @@ -23,21 +22,23 @@ import ( "github.com/coreos/go-semver/semver" "github.com/google/uuid" "github.com/pingcap/errors" - kv "github.com/pingcap/kvproto/pkg/import_kvpb" + "github.com/pingcap/kvproto/pkg/import_kvpb" "github.com/pingcap/parser/model" "github.com/pingcap/tidb/table" "go.uber.org/zap" "google.golang.org/grpc" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/lightning/common" - "github.com/pingcap/br/pkg/lightning/glue" "github.com/pingcap/br/pkg/lightning/log" - "github.com/pingcap/br/pkg/pdutil" + "github.com/pingcap/br/pkg/lightning/tikv" "github.com/pingcap/br/pkg/version" ) const ( defaultRetryBackoffTime = time.Second * 3 + writeRowsMaxRetryTimes = 3 ) var ( @@ -54,7 +55,7 @@ var ( // goroutine safe: you can share this instance and execute any method anywhere. type importer struct { conn *grpc.ClientConn - cli kv.ImportKVClient + cli import_kvpb.ImportKVClient pdAddr string tls *common.TLS @@ -63,30 +64,30 @@ type importer struct { // NewImporter creates a new connection to tikv-importer. A single connection // per tidb-lightning instance is enough. -func NewImporter(ctx context.Context, tls *common.TLS, importServerAddr string, pdAddr string) (Backend, error) { +func NewImporter(ctx context.Context, tls *common.TLS, importServerAddr string, pdAddr string) (backend.Backend, error) { conn, err := grpc.DialContext(ctx, importServerAddr, tls.ToGRPCDialOption()) if err != nil { - return MakeBackend(nil), errors.Trace(err) + return backend.MakeBackend(nil), errors.Trace(err) } - return MakeBackend(&importer{ + return backend.MakeBackend(&importer{ conn: conn, - cli: kv.NewImportKVClient(conn), + cli: import_kvpb.NewImportKVClient(conn), pdAddr: pdAddr, tls: tls, - mutationPool: sync.Pool{New: func() interface{} { return &kv.Mutation{} }}, + mutationPool: sync.Pool{New: func() interface{} { return &import_kvpb.Mutation{} }}, }), nil } // NewMockImporter creates an *unconnected* importer based on a custom // ImportKVClient. This is provided for testing only. Do not use this function // outside of tests. -func NewMockImporter(cli kv.ImportKVClient, pdAddr string) Backend { - return MakeBackend(&importer{ +func NewMockImporter(cli import_kvpb.ImportKVClient, pdAddr string) backend.Backend { + return backend.MakeBackend(&importer{ conn: nil, cli: cli, pdAddr: pdAddr, - mutationPool: sync.Pool{New: func() interface{} { return &kv.Mutation{} }}, + mutationPool: sync.Pool{New: func() interface{} { return &import_kvpb.Mutation{} }}, }) } @@ -125,7 +126,7 @@ func isIgnorableOpenCloseEngineError(err error) bool { } func (importer *importer) OpenEngine(ctx context.Context, engineUUID uuid.UUID) error { - req := &kv.OpenEngineRequest{ + req := &import_kvpb.OpenEngineRequest{ Uuid: engineUUID[:], } @@ -134,7 +135,7 @@ func (importer *importer) OpenEngine(ctx context.Context, engineUUID uuid.UUID) } func (importer *importer) CloseEngine(ctx context.Context, engineUUID uuid.UUID) error { - req := &kv.CloseEngineRequest{ + req := &import_kvpb.CloseEngineRequest{ Uuid: engineUUID[:], } @@ -150,7 +151,7 @@ func (importer *importer) Flush(_ context.Context, _ uuid.UUID) error { } func (importer *importer) ImportEngine(ctx context.Context, engineUUID uuid.UUID) error { - req := &kv.ImportEngineRequest{ + req := &import_kvpb.ImportEngineRequest{ Uuid: engineUUID[:], PdAddr: importer.pdAddr, } @@ -160,7 +161,7 @@ func (importer *importer) ImportEngine(ctx context.Context, engineUUID uuid.UUID } func (importer *importer) CleanupEngine(ctx context.Context, engineUUID uuid.UUID) error { - req := &kv.CleanupEngineRequest{ + req := &import_kvpb.CleanupEngineRequest{ Uuid: engineUUID[:], } @@ -174,12 +175,12 @@ func (importer *importer) WriteRows( tableName string, columnNames []string, ts uint64, - rows Rows, + rows kv.Rows, ) (finalErr error) { var err error outside: for _, r := range rows.SplitIntoChunks(importer.MaxChunkSize()) { - for i := 0; i < maxRetryTimes; i++ { + for i := 0; i < writeRowsMaxRetryTimes; i++ { err = importer.WriteRowsToImporter(ctx, engineUUID, ts, r) switch { case err == nil: @@ -190,7 +191,7 @@ outside: return err } } - return errors.Annotatef(err, "[%s] write rows reach max retry %d and still failed", tableName, maxRetryTimes) + return errors.Annotatef(err, "[%s] write rows reach max retry %d and still failed", tableName, writeRowsMaxRetryTimes) } return nil } @@ -199,9 +200,9 @@ func (importer *importer) WriteRowsToImporter( ctx context.Context, engineUUID uuid.UUID, ts uint64, - rows Rows, + rows kv.Rows, ) (finalErr error) { - kvs := rows.(kvPairs) + kvs := kv.KvPairsFromRows(rows) if len(kvs) == 0 { return nil } @@ -229,9 +230,9 @@ func (importer *importer) WriteRowsToImporter( }() // Bind uuid for this write request - req := &kv.WriteEngineRequest{ - Chunk: &kv.WriteEngineRequest_Head{ - Head: &kv.WriteHead{ + req := &import_kvpb.WriteEngineRequest{ + Chunk: &import_kvpb.WriteEngineRequest_Head{ + Head: &import_kvpb.WriteHead{ Uuid: engineUUID[:], }, }, @@ -241,17 +242,17 @@ func (importer *importer) WriteRowsToImporter( } // Send kv paris as write request content - mutations := make([]*kv.Mutation, len(kvs)) + mutations := make([]*import_kvpb.Mutation, len(kvs)) for i, pair := range kvs { - mutations[i] = importer.mutationPool.Get().(*kv.Mutation) - mutations[i].Op = kv.Mutation_Put + mutations[i] = importer.mutationPool.Get().(*import_kvpb.Mutation) + mutations[i].Op = import_kvpb.Mutation_Put mutations[i].Key = pair.Key mutations[i].Value = pair.Val } req.Reset() - req.Chunk = &kv.WriteEngineRequest_Batch{ - Batch: &kv.WriteBatch{ + req.Chunk = &import_kvpb.WriteEngineRequest_Batch{ + Batch: &import_kvpb.WriteBatch{ CommitTs: ts, Mutations: mutations, }, @@ -269,22 +270,22 @@ func (importer *importer) WriteRowsToImporter( return nil } -func (*importer) MakeEmptyRows() Rows { - return kvPairs(nil) +func (*importer) MakeEmptyRows() kv.Rows { + return kv.MakeRowsFromKvPairs(nil) } -func (*importer) NewEncoder(tbl table.Table, options *SessionOptions) (Encoder, error) { - return NewTableKVEncoder(tbl, options) +func (*importer) NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) { + return kv.NewTableKVEncoder(tbl, options) } func (importer *importer) CheckRequirements(ctx context.Context) error { if err := checkTiDBVersionByTLS(ctx, importer.tls, requiredMinTiDBVersion, requiredMaxTiDBVersion); err != nil { return err } - if err := checkPDVersion(ctx, importer.tls, importer.pdAddr, requiredMinPDVersion, requiredMaxPDVersion); err != nil { + if err := tikv.CheckPDVersion(ctx, importer.tls, importer.pdAddr, requiredMinPDVersion, requiredMaxPDVersion); err != nil { return err } - if err := checkTiKVVersion(ctx, importer.tls, importer.pdAddr, requiredMinTiKVVersion, requiredMaxTiKVVersion); err != nil { + if err := tikv.CheckTiKVVersion(ctx, importer.tls, importer.pdAddr, requiredMinTiKVVersion, requiredMaxTiKVVersion); err != nil { return err } return nil @@ -297,86 +298,14 @@ func checkTiDBVersionByTLS(ctx context.Context, tls *common.TLS, requiredMinVers return err } - return checkTiDBVersion(status.Version, requiredMinVersion, requiredMaxVersion) -} - -func checkTiDBVersion(versionStr string, requiredMinVersion, requiredMaxVersion semver.Version) error { - version, err := version.ExtractTiDBVersion(versionStr) - if err != nil { - return errors.Trace(err) - } - return checkVersion("TiDB", *version, requiredMinVersion, requiredMaxVersion) -} - -func checkTiDBVersionBySQL(ctx context.Context, g glue.Glue, requiredMinVersion, requiredMaxVersion semver.Version) error { - versionStr, err := g.GetSQLExecutor().ObtainStringWithLog( - ctx, - "SELECT version();", - "check TiDB version", - log.L()) - if err != nil { - return errors.Trace(err) - } - - return checkTiDBVersion(versionStr, requiredMinVersion, requiredMaxVersion) -} - -func checkPDVersion(ctx context.Context, tls *common.TLS, pdAddr string, requiredMinVersion, requiredMaxVersion semver.Version) error { - version, err := pdutil.FetchPDVersion(ctx, tls, pdAddr) - if err != nil { - return errors.Trace(err) - } - - return checkVersion("PD", *version, requiredMinVersion, requiredMaxVersion) -} - -func checkTiKVVersion(ctx context.Context, tls *common.TLS, pdAddr string, requiredMinVersion, requiredMaxVersion semver.Version) error { - return ForAllStores( - ctx, - tls.WithHost(pdAddr), - StoreStateDown, - func(c context.Context, store *Store) error { - component := fmt.Sprintf("TiKV (at %s)", store.Address) - version, err := semver.NewVersion(strings.TrimPrefix(store.Version, "v")) - if err != nil { - return errors.Annotate(err, component) - } - return checkVersion(component, *version, requiredMinVersion, requiredMaxVersion) - }, - ) -} - -func checkVersion(component string, actual, requiredMinVersion, requiredMaxVersion semver.Version) error { - // actual version must be within [requiredMinVersion, requiredMaxVersion). - if actual.Compare(requiredMinVersion) < 0 { - return errors.Errorf( - "%s version too old, required to be in [%s, %s), found '%s'", - component, - requiredMinVersion, - requiredMaxVersion, - actual, - ) - } - // Compare the major version number to make sure beta version does not pass - // the check. This is because beta version may contains incompatible - // changes. - if actual.Major >= requiredMaxVersion.Major { - return errors.Errorf( - "%s version too new, major version expected to be within [%s, %d.0.0), found '%s'", - component, - requiredMinVersion, - requiredMaxVersion.Major, - actual, - ) - } - return nil + return version.CheckTiDBVersion(status.Version, requiredMinVersion, requiredMaxVersion) } func (importer *importer) FetchRemoteTableModels(ctx context.Context, schema string) ([]*model.TableInfo, error) { - return fetchRemoteTableModelsFromTLS(ctx, importer.tls, schema) + return tikv.FetchRemoteTableModelsFromTLS(ctx, importer.tls, schema) } -func (importer *importer) EngineFileSizes() []EngineFileSize { +func (importer *importer) EngineFileSizes() []backend.EngineFileSize { return nil } @@ -392,7 +321,7 @@ func (importer *importer) ResetEngine(context.Context, uuid.UUID) error { return errors.New("cannot reset an engine in importer backend") } -func (importer *importer) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (EngineWriter, error) { +func (importer *importer) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (backend.EngineWriter, error) { return &ImporterWriter{importer: importer, engineUUID: engineUUID}, nil } @@ -405,6 +334,6 @@ func (w *ImporterWriter) Close() error { return nil } -func (w *ImporterWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows Rows) error { +func (w *ImporterWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows kv.Rows) error { return w.importer.WriteRows(ctx, w.engineUUID, tableName, columnNames, ts, rows) } diff --git a/pkg/lightning/backend/importer_test.go b/pkg/lightning/backend/importer/importer_test.go similarity index 69% rename from pkg/lightning/backend/importer_test.go rename to pkg/lightning/backend/importer/importer_test.go index 4443ede46..aaf0f5262 100644 --- a/pkg/lightning/backend/importer_test.go +++ b/pkg/lightning/backend/importer/importer_test.go @@ -11,10 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend_test +package importer import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "sync" "testing" @@ -22,11 +25,10 @@ import ( "github.com/google/uuid" . "github.com/pingcap/check" "github.com/pingcap/errors" - "github.com/pingcap/kvproto/pkg/import_kvpb" - kvpb "github.com/pingcap/kvproto/pkg/import_kvpb" - kv "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/mock" ) @@ -37,7 +39,7 @@ type importerSuite struct { mockWriter *mock.MockImportKV_WriteEngineClient ctx context.Context engineUUID []byte - engine *kv.OpenedEngine + engine *backend.OpenedEngine kvPairs kv.Rows } @@ -52,7 +54,7 @@ func (s *importerSuite) setUpTest(c *C) { s.controller = gomock.NewController(c) s.mockClient = mock.NewMockImportKVClient(s.controller) s.mockWriter = mock.NewMockImportKV_WriteEngineClient(s.controller) - importer := kv.NewMockImporter(s.mockClient, testPDAddr) + importer := NewMockImporter(s.mockClient, testPDAddr) s.ctx = context.Background() engineUUID := uuid.MustParse("7e3f3a3c-67ce-506d-af34-417ec138fbcb") @@ -69,7 +71,7 @@ func (s *importerSuite) setUpTest(c *C) { }) s.mockClient.EXPECT(). - OpenEngine(s.ctx, &import_kvpb.OpenEngineRequest{Uuid: s.engineUUID}). + OpenEngine(s.ctx, &kvpb.OpenEngineRequest{Uuid: s.engineUUID}). Return(nil, nil) var err error @@ -88,18 +90,18 @@ func (s *importerSuite) TestWriteRows(c *C) { s.mockClient.EXPECT().WriteEngine(s.ctx).Return(s.mockWriter, nil) headSendCall := s.mockWriter.EXPECT(). - Send(&import_kvpb.WriteEngineRequest{ - Chunk: &import_kvpb.WriteEngineRequest_Head{ - Head: &import_kvpb.WriteHead{Uuid: s.engineUUID}, + Send(&kvpb.WriteEngineRequest{ + Chunk: &kvpb.WriteEngineRequest_Head{ + Head: &kvpb.WriteHead{Uuid: s.engineUUID}, }, }). Return(nil) batchSendCall := s.mockWriter.EXPECT(). Send(gomock.Any()). - DoAndReturn(func(x *import_kvpb.WriteEngineRequest) error { - c.Assert(x.GetBatch().GetMutations(), DeepEquals, []*import_kvpb.Mutation{ - {Op: import_kvpb.Mutation_Put, Key: []byte("k1"), Value: []byte("v1")}, - {Op: import_kvpb.Mutation_Put, Key: []byte("k2"), Value: []byte("v2")}, + DoAndReturn(func(x *kvpb.WriteEngineRequest) error { + c.Assert(x.GetBatch().GetMutations(), DeepEquals, []*kvpb.Mutation{ + {Op: kvpb.Mutation_Put, Key: []byte("k1"), Value: []byte("v1")}, + {Op: kvpb.Mutation_Put, Key: []byte("k2"), Value: []byte("v2")}, }) return nil }). @@ -125,7 +127,7 @@ func (s *importerSuite) TestWriteHeadSendFailed(c *C) { headSendCall := s.mockWriter.EXPECT(). Send(gomock.Any()). - DoAndReturn(func(x *import_kvpb.WriteEngineRequest) error { + DoAndReturn(func(x *kvpb.WriteEngineRequest) error { c.Assert(x.GetHead(), NotNil) return errors.Annotate(context.Canceled, "fake unrecoverable write head error") }) @@ -148,13 +150,13 @@ func (s *importerSuite) TestWriteBatchSendFailed(c *C) { headSendCall := s.mockWriter.EXPECT(). Send(gomock.Any()). - DoAndReturn(func(x *import_kvpb.WriteEngineRequest) error { + DoAndReturn(func(x *kvpb.WriteEngineRequest) error { c.Assert(x.GetHead(), NotNil) return nil }) batchSendCall := s.mockWriter.EXPECT(). Send(gomock.Any()). - DoAndReturn(func(x *import_kvpb.WriteEngineRequest) error { + DoAndReturn(func(x *kvpb.WriteEngineRequest) error { c.Assert(x.GetBatch(), NotNil) return errors.Annotate(context.Canceled, "fake unrecoverable write batch error") }). @@ -178,13 +180,13 @@ func (s *importerSuite) TestWriteCloseFailed(c *C) { headSendCall := s.mockWriter.EXPECT(). Send(gomock.Any()). - DoAndReturn(func(x *import_kvpb.WriteEngineRequest) error { + DoAndReturn(func(x *kvpb.WriteEngineRequest) error { c.Assert(x.GetHead(), NotNil) return nil }) batchSendCall := s.mockWriter.EXPECT(). Send(gomock.Any()). - DoAndReturn(func(x *import_kvpb.WriteEngineRequest) error { + DoAndReturn(func(x *kvpb.WriteEngineRequest) error { c.Assert(x.GetBatch(), NotNil) return nil }). @@ -205,13 +207,13 @@ func (s *importerSuite) TestCloseImportCleanupEngine(c *C) { defer s.tearDownTest() s.mockClient.EXPECT(). - CloseEngine(s.ctx, &import_kvpb.CloseEngineRequest{Uuid: s.engineUUID}). + CloseEngine(s.ctx, &kvpb.CloseEngineRequest{Uuid: s.engineUUID}). Return(nil, nil) s.mockClient.EXPECT(). - ImportEngine(s.ctx, &import_kvpb.ImportEngineRequest{Uuid: s.engineUUID, PdAddr: testPDAddr}). + ImportEngine(s.ctx, &kvpb.ImportEngineRequest{Uuid: s.engineUUID, PdAddr: testPDAddr}). Return(nil, nil) s.mockClient.EXPECT(). - CleanupEngine(s.ctx, &import_kvpb.CleanupEngineRequest{Uuid: s.engineUUID}). + CleanupEngine(s.ctx, &kvpb.CleanupEngineRequest{Uuid: s.engineUUID}). Return(nil, nil) engine, err := s.engine.Close(s.ctx) @@ -257,3 +259,34 @@ func BenchmarkMutationPool(b *testing.B) { _ = g } + +func (s *importerSuite) TestCheckTiDBVersion(c *C) { + var version string + ctx := context.Background() + + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + c.Assert(req.URL.Path, Equals, "/status") + w.WriteHeader(http.StatusOK) + err := json.NewEncoder(w).Encode(map[string]interface{}{ + "version": version, + }) + c.Assert(err, IsNil) + })) + + tls := common.NewTLSFromMockServer(mockServer) + + version = "5.7.25-TiDB-v4.0.0" + c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), IsNil) + + version = "5.7.25-TiDB-v9999.0.0" + c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too new.*") + + version = "5.7.25-TiDB-v6.0.0" + c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too new.*") + + version = "5.7.25-TiDB-v6.0.0-beta" + c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too new.*") + + version = "5.7.25-TiDB-v1.0.0" + c.Assert(checkTiDBVersionByTLS(ctx, tls, requiredMinTiDBVersion, requiredMaxTiDBVersion), ErrorMatches, "TiDB version too old.*") +} diff --git a/pkg/lightning/backend/allocator.go b/pkg/lightning/backend/kv/allocator.go similarity index 96% rename from pkg/lightning/backend/allocator.go rename to pkg/lightning/backend/kv/allocator.go index 80ceb56da..40dd503be 100644 --- a/pkg/lightning/backend/allocator.go +++ b/pkg/lightning/backend/kv/allocator.go @@ -11,7 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +// TODO combine with the pkg/kv package outside. + +package kv import ( "sync/atomic" diff --git a/pkg/lightning/backend/session.go b/pkg/lightning/backend/kv/session.go similarity index 96% rename from pkg/lightning/backend/session.go rename to pkg/lightning/backend/kv/session.go index 036dcdd12..5744bc66c 100644 --- a/pkg/lightning/backend/session.go +++ b/pkg/lightning/backend/kv/session.go @@ -11,7 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +// TODO combine with the pkg/kv package outside. + +package kv import ( "context" @@ -186,6 +188,11 @@ type SessionOptions struct { AutoRandomSeed int64 } +// NewSession creates a new trimmed down Session matching the options. +func NewSession(options *SessionOptions) sessionctx.Context { + return newSession(options) +} + func newSession(options *SessionOptions) *session { sqlMode := options.SQLMode vars := variable.NewSessionVars() diff --git a/pkg/lightning/backend/session_test.go b/pkg/lightning/backend/kv/session_test.go similarity index 98% rename from pkg/lightning/backend/session_test.go rename to pkg/lightning/backend/kv/session_test.go index b28f7080f..c8debcdf8 100644 --- a/pkg/lightning/backend/session_test.go +++ b/pkg/lightning/backend/kv/session_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package kv import ( "testing" diff --git a/pkg/lightning/backend/sql2kv.go b/pkg/lightning/backend/kv/sql2kv.go similarity index 93% rename from pkg/lightning/backend/sql2kv.go rename to pkg/lightning/backend/kv/sql2kv.go index 4c745f490..2e9cbc136 100644 --- a/pkg/lightning/backend/sql2kv.go +++ b/pkg/lightning/backend/kv/sql2kv.go @@ -11,7 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +// TODO combine with the pkg/kv package outside. + +package kv import ( "math/rand" @@ -40,7 +42,7 @@ import ( "github.com/pingcap/br/pkg/lightning/verification" ) -var extraHandleColumnInfo = model.NewExtraHandleColInfo() +var ExtraHandleColumnInfo = model.NewExtraHandleColInfo() type genCol struct { index int @@ -169,7 +171,8 @@ func (kvcodec *tableKVEncoder) Close() { metric.KvEncoderCounter.WithLabelValues("closed").Inc() } -type rowArrayMarshaler []types.Datum +// RowArrayMarshaler wraps a slice of types.Datum for logging the content into zap. +type RowArrayMarshaler []types.Datum var kindStr = [...]string{ types.KindNull: "null", @@ -194,7 +197,7 @@ var kindStr = [...]string{ } // MarshalLogArray implements the zapcore.ArrayMarshaler interface -func (row rowArrayMarshaler) MarshalLogArray(encoder zapcore.ArrayEncoder) error { +func (row RowArrayMarshaler) MarshalLogArray(encoder zapcore.ArrayEncoder) error { for _, datum := range row { kind := datum.Kind() var str string @@ -229,7 +232,7 @@ func logKVConvertFailed(logger log.Logger, row []types.Datum, j int, colInfo *mo } logger.Error("kv convert failed", - zap.Array("original", rowArrayMarshaler(row)), + zap.Array("original", RowArrayMarshaler(row)), zap.Int("originalCol", j), zap.String("colName", colInfo.Name.O), zap.Stringer("colType", &colInfo.FieldType), @@ -247,7 +250,7 @@ func logKVConvertFailed(logger log.Logger, row []types.Datum, j int, colInfo *mo func logEvalGenExprFailed(logger log.Logger, row []types.Datum, colInfo *model.ColumnInfo, err error) error { logger.Error("kv convert failed: cannot evaluate generated column expression", - zap.Array("original", rowArrayMarshaler(row)), + zap.Array("original", RowArrayMarshaler(row)), zap.String("colName", colInfo.Name.O), log.ShortError(err), ) @@ -275,6 +278,13 @@ func MakeRowFromKvPairs(pairs []common.KvPair) Row { return kvPairs(pairs) } +// KvPairsFromRows converts a Rows instance constructed from MakeRowsFromKvPairs +// back into a slice of KvPair. This method panics if the Rows is not +// constructed in such way. +func KvPairsFromRows(rows Rows) []common.KvPair { + return []common.KvPair(rows.(kvPairs)) +} + // Encode a row of data into KV pairs. // // See comments in `(*TableRestore).initializeColumns` for the meaning of the @@ -343,12 +353,12 @@ func (kvcodec *tableKVEncoder) Encode( if common.TableHasAutoRowID(kvcodec.tbl.Meta()) { j := columnPermutation[len(cols)] if j >= 0 && j < len(row) { - value, err = table.CastValue(kvcodec.se, row[j], extraHandleColumnInfo, false, false) + value, err = table.CastValue(kvcodec.se, row[j], ExtraHandleColumnInfo, false, false) } else { value, err = types.NewIntDatum(rowID), nil } if err != nil { - return nil, logKVConvertFailed(logger, row, j, extraHandleColumnInfo, err) + return nil, logKVConvertFailed(logger, row, j, ExtraHandleColumnInfo, err) } record = append(record, value) kvcodec.tbl.RebaseAutoID(kvcodec.se, value.GetInt64(), false, autoid.RowIDAllocType) @@ -374,8 +384,8 @@ func (kvcodec *tableKVEncoder) Encode( _, err = kvcodec.tbl.AddRecord(kvcodec.se, record) if err != nil { logger.Error("kv encode failed", - zap.Array("originalRow", rowArrayMarshaler(row)), - zap.Array("convertedRow", rowArrayMarshaler(record)), + zap.Array("originalRow", RowArrayMarshaler(row)), + zap.Array("convertedRow", RowArrayMarshaler(record)), log.ShortError(err), ) return nil, errors.Trace(err) diff --git a/pkg/lightning/backend/sql2kv_test.go b/pkg/lightning/backend/kv/sql2kv_test.go similarity index 99% rename from pkg/lightning/backend/sql2kv_test.go rename to pkg/lightning/backend/kv/sql2kv_test.go index bf5fa4e6c..a74f5acc7 100644 --- a/pkg/lightning/backend/sql2kv_test.go +++ b/pkg/lightning/backend/kv/sql2kv_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package kv import ( "errors" @@ -43,7 +43,7 @@ func (s *kvSuite) TestMarshal(c *C) { minNotNull := types.Datum{} minNotNull.SetMinNotNull() encoder := zapcore.NewMapObjectEncoder() - err := encoder.AddArray("test", rowArrayMarshaler{types.NewStringDatum("1"), nullDatum, minNotNull, types.MaxValueDatum()}) + err := encoder.AddArray("test", RowArrayMarshaler{types.NewStringDatum("1"), nullDatum, minNotNull, types.MaxValueDatum()}) c.Assert(err, IsNil) c.Assert(encoder.Fields["test"], DeepEquals, []interface{}{ map[string]interface{}{"kind": "string", "val": "1"}, @@ -54,7 +54,7 @@ func (s *kvSuite) TestMarshal(c *C) { invalid := types.Datum{} invalid.SetInterface(1) - err = encoder.AddArray("bad-test", rowArrayMarshaler{minNotNull, invalid}) + err = encoder.AddArray("bad-test", RowArrayMarshaler{minNotNull, invalid}) c.Assert(err, ErrorMatches, "cannot convert.*") c.Assert(encoder.Fields["bad-test"], DeepEquals, []interface{}{ map[string]interface{}{"kind": "min", "val": "-inf"}, diff --git a/pkg/lightning/backend/kv/types.go b/pkg/lightning/backend/kv/types.go new file mode 100644 index 000000000..299f4a8cb --- /dev/null +++ b/pkg/lightning/backend/kv/types.go @@ -0,0 +1,50 @@ +// Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. + +package kv + +import ( + "github.com/pingcap/tidb/types" + + "github.com/pingcap/br/pkg/lightning/log" + "github.com/pingcap/br/pkg/lightning/verification" +) + +// Encoder encodes a row of SQL values into some opaque type which can be +// consumed by OpenEngine.WriteEncoded. +type Encoder interface { + // Close the encoder. + Close() + + // Encode encodes a row of SQL values into a backend-friendly format. + Encode( + logger log.Logger, + row []types.Datum, + rowID int64, + columnPermutation []int, + ) (Row, error) +} + +// Row represents a single encoded row. +type Row interface { + // ClassifyAndAppend separates the data-like and index-like parts of the + // encoded row, and appends these parts into the existing buffers and + // checksums. + ClassifyAndAppend( + data *Rows, + dataChecksum *verification.KVChecksum, + indices *Rows, + indexChecksum *verification.KVChecksum, + ) +} + +// Rows represents a collection of encoded rows. +type Rows interface { + // SplitIntoChunks splits the rows into multiple consecutive parts, each + // part having total byte size less than `splitSize`. The meaning of "byte + // size" should be consistent with the value used in `Row.ClassifyAndAppend`. + SplitIntoChunks(splitSize int) []Rows + + // Clear returns a new collection with empty content. It may share the + // capacity with the current instance. The typical usage is `x = x.Clear()`. + Clear() Rows +} diff --git a/pkg/lightning/backend/local.go b/pkg/lightning/backend/local/local.go similarity index 94% rename from pkg/lightning/backend/local.go rename to pkg/lightning/backend/local/local.go index db37eddbc..57bb1da27 100644 --- a/pkg/lightning/backend/local.go +++ b/pkg/lightning/backend/local/local.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package local import ( "bytes" @@ -53,12 +53,15 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/glue" "github.com/pingcap/br/pkg/lightning/log" "github.com/pingcap/br/pkg/lightning/manual" "github.com/pingcap/br/pkg/lightning/metric" + "github.com/pingcap/br/pkg/lightning/tikv" "github.com/pingcap/br/pkg/lightning/worker" split "github.com/pingcap/br/pkg/restore" "github.com/pingcap/br/pkg/utils" @@ -66,8 +69,10 @@ import ( ) const ( - dialTimeout = 5 * time.Second - bigValueSize = 1 << 16 // 64K + dialTimeout = 5 * time.Second + bigValueSize = 1 << 16 // 64K + maxRetryTimes = 3 + defaultRetryBackoffTime = 3 * time.Second gRPCKeepAliveTime = 10 * time.Second gRPCKeepAliveTimeout = 3 * time.Second @@ -137,7 +142,7 @@ const ( importMutexStateLocalIngest ) -type LocalFile struct { +type File struct { localFileMeta db *pebble.DB Uuid uuid.UUID @@ -149,7 +154,7 @@ type LocalFile struct { mutex sync.Mutex } -func (e *LocalFile) Close() error { +func (e *File) Close() error { log.L().Debug("closing local engine", zap.Stringer("engine", e.Uuid), zap.Stack("stack")) if e.db == nil { return nil @@ -160,13 +165,13 @@ func (e *LocalFile) Close() error { } // Cleanup remove meta and db files -func (e *LocalFile) Cleanup(dataDir string) error { +func (e *File) Cleanup(dataDir string) error { dbPath := filepath.Join(dataDir, e.Uuid.String()) return os.RemoveAll(dbPath) } // Exist checks if db folder existing (meta sometimes won't flush before lightning exit) -func (e *LocalFile) Exist(dataDir string) error { +func (e *File) Exist(dataDir string) error { dbPath := filepath.Join(dataDir, e.Uuid.String()) if _, err := os.Stat(dbPath); err != nil { return err @@ -174,7 +179,7 @@ func (e *LocalFile) Exist(dataDir string) error { return nil } -func (e *LocalFile) getSizeProperties() (*sizeProperties, error) { +func (e *File) getSizeProperties() (*sizeProperties, error) { sstables, err := e.db.SSTables(pebble.WithProperties()) if err != nil { log.L().Warn("get table properties failed", zap.Stringer("engine", e.Uuid), log.ShortError(err)) @@ -201,11 +206,11 @@ func (e *LocalFile) getSizeProperties() (*sizeProperties, error) { return sizeProps, nil } -func (e *LocalFile) isLocked() bool { +func (e *File) isLocked() bool { return e.isImportingAtomic.Load() != 0 } -func (e *LocalFile) getEngineFileSize() EngineFileSize { +func (e *File) getEngineFileSize() backend.EngineFileSize { metrics := e.db.Metrics() total := metrics.Total() var memSize int64 @@ -218,7 +223,7 @@ func (e *LocalFile) getEngineFileSize() EngineFileSize { return true }) - return EngineFileSize{ + return backend.EngineFileSize{ UUID: e.Uuid, DiskSize: total.Size, MemSize: memSize, @@ -227,14 +232,14 @@ func (e *LocalFile) getEngineFileSize() EngineFileSize { } // lock locks the local file for importing. -func (e *LocalFile) lock(state importMutexState) { +func (e *File) lock(state importMutexState) { e.mutex.Lock() e.isImportingAtomic.Store(uint32(state)) } // lockUnless tries to lock the local file unless it is already locked into the state given by // ignoreStateMask. Returns whether the lock is successful. -func (e *LocalFile) lockUnless(newState, ignoreStateMask importMutexState) bool { +func (e *File) lockUnless(newState, ignoreStateMask importMutexState) bool { curState := e.isImportingAtomic.Load() if curState&uint32(ignoreStateMask) != 0 { return false @@ -243,7 +248,7 @@ func (e *LocalFile) lockUnless(newState, ignoreStateMask importMutexState) bool return true } -func (e *LocalFile) unlock() { +func (e *File) unlock() { if e == nil { return } @@ -251,7 +256,7 @@ func (e *LocalFile) unlock() { e.mutex.Unlock() } -func (e *LocalFile) flushLocalWriters(parentCtx context.Context) error { +func (e *File) flushLocalWriters(parentCtx context.Context) error { eg, ctx := errgroup.WithContext(parentCtx) e.localWriters.Range(func(k, v interface{}) bool { eg.Go(func() error { @@ -276,7 +281,7 @@ func (e *LocalFile) flushLocalWriters(parentCtx context.Context) error { return eg.Wait() } -func (e *LocalFile) flushEngineWithoutLock(ctx context.Context) error { +func (e *File) flushEngineWithoutLock(ctx context.Context) error { if err := e.flushLocalWriters(ctx); err != nil { return err } @@ -297,7 +302,7 @@ func (e *LocalFile) flushEngineWithoutLock(ctx context.Context) error { // saveEngineMeta saves the metadata about the DB into the DB itself. // This method should be followed by a Flush to ensure the data is actually synchronized -func (e *LocalFile) saveEngineMeta() error { +func (e *File) saveEngineMeta() error { jsonBytes, err := json.Marshal(&e.localFileMeta) if err != nil { return errors.Trace(err) @@ -306,7 +311,7 @@ func (e *LocalFile) saveEngineMeta() error { return errors.Trace(e.db.Set(engineMetaKey, jsonBytes, &pebble.WriteOptions{Sync: false})) } -func (e *LocalFile) loadEngineMeta() { +func (e *File) loadEngineMeta() { jsonBytes, closer, err := e.db.Get(engineMetaKey) if err != nil { log.L().Debug("local db missing engine meta", zap.Stringer("uuid", e.Uuid), zap.Error(err)) @@ -335,7 +340,7 @@ func (conns *gRPCConns) Close() { } type local struct { - engines sync.Map // sync version of map[uuid.UUID]*LocalFile + engines sync.Map // sync version of map[uuid.UUID]*File conns gRPCConns splitCli split.SplitClient @@ -426,13 +431,13 @@ func NewLocalBackend( enableCheckpoint bool, g glue.Glue, maxOpenFiles int, -) (Backend, error) { +) (backend.Backend, error) { localFile := cfg.SortedKVDir rangeConcurrency := cfg.RangeConcurrency pdCli, err := pd.NewClientWithContext(ctx, []string{pdAddr}, tls.ToPDSecurityOption()) if err != nil { - return MakeBackend(nil), errors.Annotate(err, "construct pd client failed") + return backend.MakeBackend(nil), errors.Annotate(err, "construct pd client failed") } splitCli := split.NewSplitClient(pdCli, tls.TLSConfig()) @@ -440,7 +445,7 @@ func NewLocalBackend( if enableCheckpoint { if info, err := os.Stat(localFile); err != nil { if !os.IsNotExist(err) { - return MakeBackend(nil), err + return backend.MakeBackend(nil), err } } else if info.IsDir() { shouldCreate = false @@ -450,7 +455,7 @@ func NewLocalBackend( if shouldCreate { err = os.Mkdir(localFile, 0o700) if err != nil { - return MakeBackend(nil), errors.Annotate(err, "invalid sorted-kv-dir for local backend, please change the config or delete the path") + return backend.MakeBackend(nil), errors.Annotate(err, "invalid sorted-kv-dir for local backend, please change the config or delete the path") } } @@ -475,13 +480,13 @@ func NewLocalBackend( localWriterMemCacheSize: int64(cfg.LocalWriterMemCacheSize), } local.conns.conns = make(map[uint64]*connPool) - return MakeBackend(local), nil + return backend.MakeBackend(local), nil } -// lock locks a local file and returns the LocalFile instance if it exists. -func (local *local) lockEngine(engineId uuid.UUID, state importMutexState) *LocalFile { +// lock locks a local file and returns the File instance if it exists. +func (local *local) lockEngine(engineId uuid.UUID, state importMutexState) *File { if e, ok := local.engines.Load(engineId); ok { - engine := e.(*LocalFile) + engine := e.(*File) engine.lock(state) return engine } @@ -490,10 +495,10 @@ func (local *local) lockEngine(engineId uuid.UUID, state importMutexState) *Loca // lockAllEnginesUnless tries to lock all engines, unless those which are already locked in the // state given by ignoreStateMask. Returns the list of locked engines. -func (local *local) lockAllEnginesUnless(newState, ignoreStateMask importMutexState) []*LocalFile { - var allEngines []*LocalFile +func (local *local) lockAllEnginesUnless(newState, ignoreStateMask importMutexState) []*File { + var allEngines []*File local.engines.Range(func(k, v interface{}) bool { - engine := v.(*LocalFile) + engine := v.(*File) if engine.lockUnless(newState, ignoreStateMask) { allEngines = append(allEngines, engine) } @@ -633,14 +638,14 @@ func (local *local) openEngineDB(engineUUID uuid.UUID, readOnly bool) (*pebble.D return pebble.Open(dbPath, opt) } -// This method must be called with holding mutex of LocalFile +// This method must be called with holding mutex of File func (local *local) OpenEngine(ctx context.Context, engineUUID uuid.UUID) error { db, err := local.openEngineDB(engineUUID, false) if err != nil { return err } - e, _ := local.engines.LoadOrStore(engineUUID, &LocalFile{Uuid: engineUUID}) - engine := e.(*LocalFile) + e, _ := local.engines.LoadOrStore(engineUUID, &File{Uuid: engineUUID}) + engine := e.(*File) engine.db = db engine.loadEngineMeta() return nil @@ -663,7 +668,7 @@ func (local *local) CloseEngine(ctx context.Context, engineUUID uuid.UUID) error } return err } - engineFile := &LocalFile{ + engineFile := &File{ Uuid: engineUUID, db: db, } @@ -671,7 +676,7 @@ func (local *local) CloseEngine(ctx context.Context, engineUUID uuid.UUID) error local.engines.Store(engineUUID, engineFile) return nil } - engineFile := engine.(*LocalFile) + engineFile := engine.(*File) engineFile.lock(importMutexStateFlush) defer engineFile.unlock() return engineFile.flushEngineWithoutLock(ctx) @@ -698,7 +703,7 @@ type rangeStats struct { // tikv will takes the responsibility to do so. func (local *local) WriteToTiKV( ctx context.Context, - engineFile *LocalFile, + engineFile *File, region *split.RegionInfo, start, end []byte, ) ([]*sst.SSTMeta, *Range, rangeStats, error) { @@ -919,7 +924,7 @@ func splitRangeBySizeProps(fullRange Range, sizeProps *sizeProperties, sizeLimit return ranges } -func (local *local) readAndSplitIntoRange(engineFile *LocalFile) ([]Range, error) { +func (local *local) readAndSplitIntoRange(engineFile *File) ([]Range, error) { iter := engineFile.db.NewIter(&pebble.IterOptions{LowerBound: normalIterStartKey}) defer iter.Close() @@ -1067,7 +1072,7 @@ func (b *bytesBuffer) addBytes(bytes []byte) []byte { func (local *local) writeAndIngestByRange( ctxt context.Context, - engineFile *LocalFile, + engineFile *File, start, end []byte, remainRanges *syncdRanges, ) error { @@ -1167,7 +1172,7 @@ const ( func (local *local) writeAndIngestPairs( ctx context.Context, - engineFile *LocalFile, + engineFile *File, region *split.RegionInfo, start, end []byte, ) (*Range, error) { @@ -1266,7 +1271,7 @@ loopWrite: return remainRange, errors.Trace(err) } -func (local *local) writeAndIngestByRanges(ctx context.Context, engineFile *LocalFile, ranges []Range, remainRanges *syncdRanges) error { +func (local *local) writeAndIngestByRanges(ctx context.Context, engineFile *File, ranges []Range, remainRanges *syncdRanges) error { if engineFile.Length.Load() == 0 { // engine is empty, this is likes because it's a index engine but the table contains no index log.L().Info("engine contains no data", zap.Stringer("uuid", engineFile.Uuid)) @@ -1454,37 +1459,50 @@ func (local *local) CheckRequirements(ctx context.Context) error { if err := checkTiDBVersionBySQL(ctx, local.g, localMinTiDBVersion, localMaxTiDBVersion); err != nil { return err } - if err := checkPDVersion(ctx, local.tls, local.pdAddr, localMinPDVersion, localMaxPDVersion); err != nil { + if err := tikv.CheckPDVersion(ctx, local.tls, local.pdAddr, localMinPDVersion, localMaxPDVersion); err != nil { return err } - if err := checkTiKVVersion(ctx, local.tls, local.pdAddr, localMinTiKVVersion, localMaxTiKVVersion); err != nil { + if err := tikv.CheckTiKVVersion(ctx, local.tls, local.pdAddr, localMinTiKVVersion, localMaxTiKVVersion); err != nil { return err } return nil } +func checkTiDBVersionBySQL(ctx context.Context, g glue.Glue, requiredMinVersion, requiredMaxVersion semver.Version) error { + versionStr, err := g.GetSQLExecutor().ObtainStringWithLog( + ctx, + "SELECT version();", + "check TiDB version", + log.L()) + if err != nil { + return errors.Trace(err) + } + + return version.CheckTiDBVersion(versionStr, requiredMinVersion, requiredMaxVersion) +} + func (local *local) FetchRemoteTableModels(ctx context.Context, schemaName string) ([]*model.TableInfo, error) { - return fetchRemoteTableModelsFromTLS(ctx, local.tls, schemaName) + return tikv.FetchRemoteTableModelsFromTLS(ctx, local.tls, schemaName) } -func (local *local) MakeEmptyRows() Rows { - return kvPairs(nil) +func (local *local) MakeEmptyRows() kv.Rows { + return kv.MakeRowsFromKvPairs(nil) } -func (local *local) NewEncoder(tbl table.Table, options *SessionOptions) (Encoder, error) { - return NewTableKVEncoder(tbl, options) +func (local *local) NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) { + return kv.NewTableKVEncoder(tbl, options) } -func (local *local) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (EngineWriter, error) { +func (local *local) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (backend.EngineWriter, error) { e, ok := local.engines.Load(engineUUID) if !ok { return nil, errors.Errorf("could not find engine for %s", engineUUID.String()) } - engineFile := e.(*LocalFile) + engineFile := e.(*File) return openLocalWriter(engineFile, local.localStoreDir, local.localWriterMemCacheSize), nil } -func openLocalWriter(f *LocalFile, sstDir string, memtableSizeLimit int64) *LocalWriter { +func openLocalWriter(f *File, sstDir string, memtableSizeLimit int64) *LocalWriter { w := &LocalWriter{ sstDir: sstDir, kvsChan: make(chan []common.KvPair, defaultLocalWriterKVsChannelCap), @@ -1774,9 +1792,9 @@ func (s *sizeProperties) iter(f func(p *rangeProperty) bool) { }) } -func (local *local) EngineFileSizes() (res []EngineFileSize) { +func (local *local) EngineFileSizes() (res []backend.EngineFileSize) { local.engines.Range(func(k, v interface{}) bool { - engine := v.(*LocalFile) + engine := v.(*File) res = append(res, engine.getEngineFileSize()) return true }) @@ -1785,7 +1803,7 @@ func (local *local) EngineFileSizes() (res []EngineFileSize) { type LocalWriter struct { writeErr common.OnceError - local *LocalFile + local *File consumeCh chan struct{} kvsChan chan []common.KvPair flushChMutex sync.RWMutex @@ -1796,8 +1814,8 @@ type LocalWriter struct { writer *sstWriter } -func (w *LocalWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows Rows) error { - kvs := rows.(kvPairs) +func (w *LocalWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows kv.Rows) error { + kvs := kv.KvPairsFromRows(rows) if len(kvs) == 0 { return nil } @@ -1973,9 +1991,9 @@ func (sw *sstWriter) writeKVs(m *kvMemCache) error { return nil } -// ingestInto finishes the SST file, and ingests itself into the target LocalFile database. +// ingestInto finishes the SST file, and ingests itself into the target File database. // On success, the entire writer will be reset as empty. -func (sw *sstWriter) ingestInto(e *LocalFile, desc localIngestDescription) error { +func (sw *sstWriter) ingestInto(e *File, desc localIngestDescription) error { if sw.totalCount > 0 { if err := sw.writer.Close(); err != nil { return errors.Trace(err) diff --git a/pkg/lightning/backend/local_test.go b/pkg/lightning/backend/local/local_test.go similarity index 94% rename from pkg/lightning/backend/local_test.go rename to pkg/lightning/backend/local/local_test.go index cdc51ef0c..943eb7069 100644 --- a/pkg/lightning/backend/local_test.go +++ b/pkg/lightning/backend/local/local_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package local import ( "bytes" @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "sort" + "testing" "github.com/cockroachdb/pebble" "github.com/docker/go-units" @@ -29,13 +30,15 @@ import ( "github.com/pingcap/kvproto/pkg/errorpb" sst "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/tidb/kv" + tidbkv "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/sessionctx/stmtctx" "github.com/pingcap/tidb/tablecodec" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/codec" "github.com/pingcap/tidb/util/hack" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/restore" ) @@ -44,6 +47,10 @@ type localSuite struct{} var _ = Suite(&localSuite{}) +func Test(t *testing.T) { + TestingT(t) +} + func (s *localSuite) TestNextKey(c *C) { c.Assert(nextKey([]byte{}), DeepEquals, []byte{}) @@ -68,8 +75,8 @@ func (s *localSuite) TestNextKey(c *C) { // test recode key // key with int handle for _, handleId := range []int64{1, 255, math.MaxInt32} { - key := tablecodec.EncodeRowKeyWithHandle(1, kv.IntHandle(handleId)) - c.Assert(nextKey(key), DeepEquals, []byte(tablecodec.EncodeRowKeyWithHandle(1, kv.IntHandle(handleId+1)))) + key := tablecodec.EncodeRowKeyWithHandle(1, tidbkv.IntHandle(handleId)) + c.Assert(nextKey(key), DeepEquals, []byte(tablecodec.EncodeRowKeyWithHandle(1, tidbkv.IntHandle(handleId+1)))) } testDatums := [][]types.Datum{ @@ -84,12 +91,12 @@ func (s *localSuite) TestNextKey(c *C) { for _, datums := range testDatums { keyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewIntDatum(123), datums[0]) c.Assert(err, IsNil) - h, err := kv.NewCommonHandle(keyBytes) + h, err := tidbkv.NewCommonHandle(keyBytes) c.Assert(err, IsNil) key := tablecodec.EncodeRowKeyWithHandle(1, h) nextKeyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewIntDatum(123), datums[1]) c.Assert(err, IsNil) - nextHdl, err := kv.NewCommonHandle(nextKeyBytes) + nextHdl, err := tidbkv.NewCommonHandle(nextKeyBytes) c.Assert(err, IsNil) expectNextKey := []byte(tablecodec.EncodeRowKeyWithHandle(1, nextHdl)) c.Assert(nextKey(key), DeepEquals, expectNextKey) @@ -312,13 +319,12 @@ func testLocalWriter(c *C, needSort bool, partitialSort bool) { err = os.Mkdir(tmpPath, 0o755) c.Assert(err, IsNil) meta := localFileMeta{} - _, engineUUID := MakeUUID("ww", 0) - f := LocalFile{localFileMeta: meta, db: db, Uuid: engineUUID} + _, engineUUID := backend.MakeUUID("ww", 0) + f := File{localFileMeta: meta, db: db, Uuid: engineUUID} w := openLocalWriter(&f, tmpPath, 1024*1024) ctx := context.Background() - // kvs := make(kvPairs, 1000) - var kvs kvPairs + var kvs []common.KvPair value := make([]byte, 128) for i := 0; i < 16; i++ { binary.BigEndian.PutUint64(value[i*8:], uint64(i)) @@ -335,9 +341,9 @@ func testLocalWriter(c *C, needSort bool, partitialSort bool) { kvs = append(kvs, kv) keys = append(keys, kv.Key) } - var rows1 kvPairs - var rows2 kvPairs - var rows3 kvPairs + var rows1 []common.KvPair + var rows2 []common.KvPair + var rows3 []common.KvPair rows4 := kvs[:12000] if partitialSort { sort.Slice(rows4, func(i, j int) bool { @@ -356,11 +362,11 @@ func testLocalWriter(c *C, needSort bool, partitialSort bool) { rows2 = kvs[6000:12000] rows3 = kvs[12000:] } - err = w.AppendRows(ctx, "", []string{}, 1, rows1) + err = w.AppendRows(ctx, "", []string{}, 1, kv.MakeRowsFromKvPairs(rows1)) c.Assert(err, IsNil) - err = w.AppendRows(ctx, "", []string{}, 1, rows2) + err = w.AppendRows(ctx, "", []string{}, 1, kv.MakeRowsFromKvPairs(rows2)) c.Assert(err, IsNil) - err = w.AppendRows(ctx, "", []string{}, 1, rows3) + err = w.AppendRows(ctx, "", []string{}, 1, kv.MakeRowsFromKvPairs(rows3)) c.Assert(err, IsNil) err = w.Close() c.Assert(err, IsNil) diff --git a/pkg/lightning/backend/local_unix.go b/pkg/lightning/backend/local/local_unix.go similarity index 99% rename from pkg/lightning/backend/local_unix.go rename to pkg/lightning/backend/local/local_unix.go index f6ff59130..8fda30b37 100644 --- a/pkg/lightning/backend/local_unix.go +++ b/pkg/lightning/backend/local/local_unix.go @@ -13,7 +13,7 @@ // +build !windows -package backend +package local import ( "syscall" diff --git a/pkg/lightning/backend/local_windows.go b/pkg/lightning/backend/local/local_windows.go similarity index 98% rename from pkg/lightning/backend/local_windows.go rename to pkg/lightning/backend/local/local_windows.go index 670f83e7d..d746ff4a6 100644 --- a/pkg/lightning/backend/local_windows.go +++ b/pkg/lightning/backend/local/local_windows.go @@ -13,7 +13,7 @@ // +build windows -package backend +package local import ( "math" diff --git a/pkg/lightning/backend/localhelper.go b/pkg/lightning/backend/local/localhelper.go similarity index 99% rename from pkg/lightning/backend/localhelper.go rename to pkg/lightning/backend/local/localhelper.go index c1c497686..411f1cb84 100644 --- a/pkg/lightning/backend/localhelper.go +++ b/pkg/lightning/backend/local/localhelper.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package local import ( "bytes" diff --git a/pkg/lightning/backend/localhelper_test.go b/pkg/lightning/backend/local/localhelper_test.go similarity index 99% rename from pkg/lightning/backend/localhelper_test.go rename to pkg/lightning/backend/local/localhelper_test.go index aec318667..f754b8635 100644 --- a/pkg/lightning/backend/localhelper_test.go +++ b/pkg/lightning/backend/local/localhelper_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package local import ( "bytes" diff --git a/pkg/lightning/backend/tidb.go b/pkg/lightning/backend/tidb/tidb.go similarity index 90% rename from pkg/lightning/backend/tidb.go rename to pkg/lightning/backend/tidb/tidb.go index 602a56411..8066b9df6 100644 --- a/pkg/lightning/backend/tidb.go +++ b/pkg/lightning/backend/tidb/tidb.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package tidb import ( "context" @@ -27,11 +27,14 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/parser/model" "github.com/pingcap/parser/mysql" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/table" "github.com/pingcap/tidb/types" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/log" @@ -40,11 +43,15 @@ import ( ) var extraHandleTableColumn = &table.Column{ - ColumnInfo: extraHandleColumnInfo, + ColumnInfo: kv.ExtraHandleColumnInfo, GeneratedExpr: nil, DefaultExpr: nil, } +const ( + writeRowsMaxRetryTimes = 3 +) + type tidbRow string type tidbRows []tidbRow @@ -60,7 +67,7 @@ func (row tidbRows) MarshalLogArray(encoder zapcore.ArrayEncoder) error { type tidbEncoder struct { mode mysql.SQLMode tbl table.Table - se *session + se sessionctx.Context // the index of table columns for each data field. // index == len(table.columns) means this field is `_tidb_rowid` columnIdx []int @@ -76,29 +83,29 @@ type tidbBackend struct { // // The backend does not take ownership of `db`. Caller should close `db` // manually after the backend expired. -func NewTiDBBackend(db *sql.DB, onDuplicate string) Backend { +func NewTiDBBackend(db *sql.DB, onDuplicate string) backend.Backend { switch onDuplicate { case config.ReplaceOnDup, config.IgnoreOnDup, config.ErrorOnDup: default: log.L().Warn("unsupported action on duplicate, overwrite with `replace`") onDuplicate = config.ReplaceOnDup } - return MakeBackend(&tidbBackend{db: db, onDuplicate: onDuplicate}) + return backend.MakeBackend(&tidbBackend{db: db, onDuplicate: onDuplicate}) } -func (row tidbRow) ClassifyAndAppend(data *Rows, checksum *verification.KVChecksum, _ *Rows, _ *verification.KVChecksum) { +func (row tidbRow) ClassifyAndAppend(data *kv.Rows, checksum *verification.KVChecksum, _ *kv.Rows, _ *verification.KVChecksum) { rows := (*data).(tidbRows) *data = tidbRows(append(rows, row)) cs := verification.MakeKVChecksum(uint64(len(row)), 1, 0) checksum.Add(&cs) } -func (rows tidbRows) SplitIntoChunks(splitSize int) []Rows { +func (rows tidbRows) SplitIntoChunks(splitSize int) []kv.Rows { if len(rows) == 0 { return nil } - res := make([]Rows, 0, 1) + res := make([]kv.Rows, 0, 1) i := 0 cumSize := 0 @@ -114,7 +121,7 @@ func (rows tidbRows) SplitIntoChunks(splitSize int) []Rows { return append(res, rows[i:]) } -func (rows tidbRows) Clear() Rows { +func (rows tidbRows) Clear() kv.Rows { return rows[:0] } @@ -246,7 +253,7 @@ func getColumnByIndex(cols []*table.Column, index int) *table.Column { return cols[index] } -func (enc *tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, columnPermutation []int) (Row, error) { +func (enc *tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, columnPermutation []int) (kv.Row, error) { cols := enc.tbl.Cols() if len(enc.columnIdx) == 0 { @@ -267,7 +274,7 @@ func (enc *tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, co // column permutation with default, thus enc.columnCnt > len(row). if len(row) > enc.columnCnt { logger.Error("column count mismatch", zap.Ints("column_permutation", columnPermutation), - zap.Array("data", rowArrayMarshaler(row))) + zap.Array("data", kv.RowArrayMarshaler(row))) return nil, errors.Errorf("column count mismatch, expected %d, got %d", enc.columnCnt, len(row)) } @@ -280,7 +287,7 @@ func (enc *tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, co } if err := enc.appendSQL(&encoded, &field, getColumnByIndex(cols, enc.columnIdx[i])); err != nil { logger.Error("tidb encode failed", - zap.Array("original", rowArrayMarshaler(row)), + zap.Array("original", kv.RowArrayMarshaler(row)), zap.Int("originalCol", i), log.ShortError(err), ) @@ -296,7 +303,7 @@ func (be *tidbBackend) Close() { // TidbManager, so we let the manager to close it. } -func (be *tidbBackend) MakeEmptyRows() Rows { +func (be *tidbBackend) MakeEmptyRows() kv.Rows { return tidbRows(nil) } @@ -320,11 +327,11 @@ func (be *tidbBackend) CheckRequirements(ctx context.Context) error { return nil } -func (be *tidbBackend) NewEncoder(tbl table.Table, options *SessionOptions) (Encoder, error) { - se := newSession(options) +func (be *tidbBackend) NewEncoder(tbl table.Table, options *kv.SessionOptions) (kv.Encoder, error) { + se := kv.NewSession(options) if options.SQLMode.HasStrictMode() { - se.vars.SkipUTF8Check = false - se.vars.SkipASCIICheck = false + se.GetSessionVars().SkipUTF8Check = false + se.GetSessionVars().SkipASCIICheck = false } return &tidbEncoder{mode: options.SQLMode, tbl: tbl, se: se}, nil @@ -346,11 +353,11 @@ func (be *tidbBackend) ImportEngine(context.Context, uuid.UUID) error { return nil } -func (be *tidbBackend) WriteRows(ctx context.Context, _ uuid.UUID, tableName string, columnNames []string, _ uint64, rows Rows) error { +func (be *tidbBackend) WriteRows(ctx context.Context, _ uuid.UUID, tableName string, columnNames []string, _ uint64, rows kv.Rows) error { var err error outside: for _, r := range rows.SplitIntoChunks(be.MaxChunkSize()) { - for i := 0; i < maxRetryTimes; i++ { + for i := 0; i < writeRowsMaxRetryTimes; i++ { err = be.WriteRowsToDB(ctx, tableName, columnNames, r) switch { case err == nil: @@ -361,12 +368,12 @@ outside: return err } } - return errors.Annotatef(err, "[%s] write rows reach max retry %d and still failed", tableName, maxRetryTimes) + return errors.Annotatef(err, "[%s] write rows reach max retry %d and still failed", tableName, writeRowsMaxRetryTimes) } return nil } -func (be *tidbBackend) WriteRowsToDB(ctx context.Context, tableName string, columnNames []string, r Rows) error { +func (be *tidbBackend) WriteRowsToDB(ctx context.Context, tableName string, columnNames []string, r kv.Rows) error { rows := r.(tidbRows) if len(rows) == 0 { return nil @@ -549,7 +556,7 @@ func (be *tidbBackend) FetchRemoteTableModels(ctx context.Context, schemaName st return } -func (be *tidbBackend) EngineFileSizes() []EngineFileSize { +func (be *tidbBackend) EngineFileSizes() []backend.EngineFileSize { return nil } @@ -565,7 +572,7 @@ func (be *tidbBackend) ResetEngine(context.Context, uuid.UUID) error { return errors.New("cannot reset an engine in TiDB backend") } -func (be *tidbBackend) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (EngineWriter, error) { +func (be *tidbBackend) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (backend.EngineWriter, error) { return &TiDBWriter{be: be, engineUUID: engineUUID}, nil } @@ -578,6 +585,6 @@ func (w *TiDBWriter) Close() error { return nil } -func (w *TiDBWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, arg1 uint64, rows Rows) error { +func (w *TiDBWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, arg1 uint64, rows kv.Rows) error { return w.be.WriteRows(ctx, w.engineUUID, tableName, columnNames, arg1, rows) } diff --git a/pkg/lightning/backend/tidb_test.go b/pkg/lightning/backend/tidb/tidb_test.go similarity index 94% rename from pkg/lightning/backend/tidb_test.go rename to pkg/lightning/backend/tidb/tidb_test.go index 4603419be..785e1b9dc 100644 --- a/pkg/lightning/backend/tidb_test.go +++ b/pkg/lightning/backend/tidb/tidb_test.go @@ -11,12 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend_test +package tidb_test import ( "context" "database/sql" "fmt" + "testing" "github.com/pingcap/parser/charset" @@ -28,18 +29,24 @@ import ( "github.com/pingcap/tidb/table/tables" "github.com/pingcap/tidb/types" - kv "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/kv" + "github.com/pingcap/br/pkg/lightning/backend/tidb" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/log" "github.com/pingcap/br/pkg/lightning/verification" ) +func Test(t *testing.T) { + TestingT(t) +} + var _ = Suite(&mysqlSuite{}) type mysqlSuite struct { dbHandle *sql.DB mockDB sqlmock.Sqlmock - backend kv.Backend + backend backend.Backend tbl table.Table } @@ -62,7 +69,7 @@ func (s *mysqlSuite) SetUpTest(c *C) { s.dbHandle = db s.mockDB = mock - s.backend = kv.NewTiDBBackend(db, config.ReplaceOnDup) + s.backend = tidb.NewTiDBBackend(db, config.ReplaceOnDup) s.tbl = tbl } @@ -129,7 +136,7 @@ func (s *mysqlSuite) TestWriteRowsIgnoreOnDup(c *C) { ctx := context.Background() logger := log.L() - ignoreBackend := kv.NewTiDBBackend(s.dbHandle, config.IgnoreOnDup) + ignoreBackend := tidb.NewTiDBBackend(s.dbHandle, config.IgnoreOnDup) engine, err := ignoreBackend.OpenEngine(ctx, "`foo`.`bar`", 1, 0) c.Assert(err, IsNil) @@ -171,7 +178,7 @@ func (s *mysqlSuite) TestWriteRowsErrorOnDup(c *C) { ctx := context.Background() logger := log.L() - ignoreBackend := kv.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) + ignoreBackend := tidb.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) engine, err := ignoreBackend.OpenEngine(ctx, "`foo`.`bar`", 1, 0) c.Assert(err, IsNil) @@ -209,7 +216,7 @@ func (s *mysqlSuite) testStrictMode(c *C) { tbl, err := tables.TableFromMeta(kv.NewPanickingAllocators(0), tblInfo) c.Assert(err, IsNil) - bk := kv.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) + bk := tidb.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) encoder, err := bk.NewEncoder(tbl, &kv.SessionOptions{SQLMode: mysql.ModeStrictAllTables}) c.Assert(err, IsNil) @@ -244,7 +251,7 @@ func (s *mysqlSuite) TestFetchRemoteTableModels_3_x(c *C) { AddRow("t", "id", "int(10)", "auto_increment")) s.mockDB.ExpectCommit() - bk := kv.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) + bk := tidb.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) tableInfos, err := bk.FetchRemoteTableModels(context.Background(), "test") c.Assert(err, IsNil) c.Assert(tableInfos, DeepEquals, []*model.TableInfo{ @@ -279,7 +286,7 @@ func (s *mysqlSuite) TestFetchRemoteTableModels_4_0(c *C) { AddRow("test", "t", "id", int64(1))) s.mockDB.ExpectCommit() - bk := kv.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) + bk := tidb.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) tableInfos, err := bk.FetchRemoteTableModels(context.Background(), "test") c.Assert(err, IsNil) c.Assert(tableInfos, DeepEquals, []*model.TableInfo{ @@ -314,7 +321,7 @@ func (s *mysqlSuite) TestFetchRemoteTableModels_4_x_auto_increment(c *C) { AddRow("test", "t", "id", int64(1), "AUTO_INCREMENT")) s.mockDB.ExpectCommit() - bk := kv.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) + bk := tidb.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) tableInfos, err := bk.FetchRemoteTableModels(context.Background(), "test") c.Assert(err, IsNil) c.Assert(tableInfos, DeepEquals, []*model.TableInfo{ @@ -349,7 +356,7 @@ func (s *mysqlSuite) TestFetchRemoteTableModels_4_x_auto_random(c *C) { AddRow("test", "t", "id", int64(1), "AUTO_RANDOM")) s.mockDB.ExpectCommit() - bk := kv.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) + bk := tidb.NewTiDBBackend(s.dbHandle, config.ErrorOnDup) tableInfos, err := bk.FetchRemoteTableModels(context.Background(), "test") c.Assert(err, IsNil) c.Assert(tableInfos, DeepEquals, []*model.TableInfo{ diff --git a/pkg/lightning/backend/tikv_test.go b/pkg/lightning/backend/tikv_test.go deleted file mode 100644 index a74b42c79..000000000 --- a/pkg/lightning/backend/tikv_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package backend_test - -import ( - "context" - "net/http" - "net/http/httptest" - "sort" - "sync" - - . "github.com/pingcap/check" - "github.com/pingcap/kvproto/pkg/import_sstpb" - - kv "github.com/pingcap/br/pkg/lightning/backend" - "github.com/pingcap/br/pkg/lightning/common" -) - -type tikvSuite struct{} - -var _ = Suite(&tikvSuite{}) - -func (s *tikvSuite) TestForAllStores(c *C) { - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.Write([]byte(` - { - "count": 5, - "stores": [ - { - "store": { - "id": 1, - "address": "127.0.0.1:20160", - "version": "3.0.0-beta.1", - "state_name": "Up" - }, - "status": {} - }, - { - "store": { - "id": 2, - "address": "127.0.0.1:20161", - "version": "3.0.0-rc.1", - "state_name": "Down" - }, - "status": {} - }, - { - "store": { - "id": 3, - "address": "127.0.0.1:20162", - "version": "3.0.0-rc.2", - "state_name": "Disconnected" - }, - "status": {} - }, - { - "store": { - "id": 4, - "address": "127.0.0.1:20163", - "version": "3.0.0", - "state_name": "Tombstone" - }, - "status": {} - }, - { - "store": { - "id": 5, - "address": "127.0.0.1:20164", - "version": "3.0.1", - "state_name": "Offline" - }, - "status": {} - } - ] - } - `)) - })) - defer server.Close() - - ctx := context.Background() - var ( - allStoresLock sync.Mutex - allStores []*kv.Store - ) - tls := common.NewTLSFromMockServer(server) - err := kv.ForAllStores(ctx, tls, kv.StoreStateDown, func(c2 context.Context, store *kv.Store) error { - allStoresLock.Lock() - allStores = append(allStores, store) - allStoresLock.Unlock() - return nil - }) - c.Assert(err, IsNil) - - sort.Slice(allStores, func(i, j int) bool { return allStores[i].Address < allStores[j].Address }) - - c.Assert(allStores, DeepEquals, []*kv.Store{ - { - Address: "127.0.0.1:20160", - Version: "3.0.0-beta.1", - State: kv.StoreStateUp, - }, - { - Address: "127.0.0.1:20161", - Version: "3.0.0-rc.1", - State: kv.StoreStateDown, - }, - { - Address: "127.0.0.1:20162", - Version: "3.0.0-rc.2", - State: kv.StoreStateDisconnected, - }, - { - Address: "127.0.0.1:20164", - Version: "3.0.1", - State: kv.StoreStateOffline, - }, - }) -} - -func (s *tikvSuite) TestFetchModeFromMetrics(c *C) { - testCases := []struct { - metrics string - mode import_sstpb.SwitchMode - isErr bool - }{ - { - metrics: `tikv_config_rocksdb{cf="default",name="hard_pending_compaction_bytes_limit"} 274877906944`, - mode: import_sstpb.SwitchMode_Normal, - }, - { - metrics: `tikv_config_rocksdb{cf="default",name="hard_pending_compaction_bytes_limit"} 0`, - mode: import_sstpb.SwitchMode_Import, - }, - { - metrics: ``, - isErr: true, - }, - } - - for _, tc := range testCases { - comment := Commentf("test case '%s'", tc.metrics) - mode, err := kv.FetchModeFromMetrics(tc.metrics) - if tc.isErr { - c.Assert(err, NotNil, comment) - } else { - c.Assert(err, IsNil, comment) - c.Assert(mode, Equals, tc.mode, comment) - } - } -} diff --git a/pkg/lightning/lightning.go b/pkg/lightning/lightning.go index 2b81843d0..d9714f0bb 100755 --- a/pkg/lightning/lightning.go +++ b/pkg/lightning/lightning.go @@ -37,7 +37,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" - "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/local" "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" @@ -666,7 +666,7 @@ func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta // region-concurrency: number of LocalWriters writing SST files. // 2*totalSize/memCacheSize: number of Pebble MemCache files. estimateMaxFiles := uint64(cfg.App.RegionConcurrency) + uint64(topNTotalSize)/uint64(cfg.TikvImporter.EngineMemCacheSize)*2 - if err := backend.VerifyRLimit(estimateMaxFiles); err != nil { + if err := local.VerifyRLimit(estimateMaxFiles); err != nil { return err } } diff --git a/pkg/lightning/lightning_test.go b/pkg/lightning/lightning_test.go index 897422aef..e79d82b29 100644 --- a/pkg/lightning/lightning_test.go +++ b/pkg/lightning/lightning_test.go @@ -469,11 +469,11 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) { } // with max open files 1024, the max table size will be: 65536MB - err := failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/GetRlimitValue", "return(2049)") + err := failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(2049)") c.Assert(err, IsNil) - err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/SetRlimitError", "return(true)") + err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/SetRlimitError", "return(true)") c.Assert(err, IsNil) - defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/SetRlimitError") + defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/SetRlimitError") // with this dbMetas, the estimated fds will be 2050, so should return error err = checkSystemRequirement(cfg, dbMetas) c.Assert(err, NotNil) @@ -484,12 +484,12 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) { c.Assert(err, IsNil) cfg.App.CheckRequirements = true - err = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/GetRlimitValue") + err = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue") c.Assert(err, IsNil) // the min rlimit should be bigger than the default min value (16384) - err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/GetRlimitValue", "return(8200)") - defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/GetRlimitValue") + err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(8200)") + defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue") c.Assert(err, IsNil) err = checkSystemRequirement(cfg, dbMetas) c.Assert(err, IsNil) diff --git a/pkg/lightning/restore/restore.go b/pkg/lightning/restore/restore.go index 7b03782ac..b0866b085 100644 --- a/pkg/lightning/restore/restore.go +++ b/pkg/lightning/restore/restore.go @@ -38,7 +38,11 @@ import ( "go.uber.org/zap" "modernc.org/mathutil" - kv "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/importer" + "github.com/pingcap/br/pkg/lightning/backend/kv" + "github.com/pingcap/br/pkg/lightning/backend/local" + "github.com/pingcap/br/pkg/lightning/backend/tidb" . "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" @@ -46,6 +50,7 @@ import ( "github.com/pingcap/br/pkg/lightning/log" "github.com/pingcap/br/pkg/lightning/metric" "github.com/pingcap/br/pkg/lightning/mydump" + "github.com/pingcap/br/pkg/lightning/tikv" verify "github.com/pingcap/br/pkg/lightning/verification" "github.com/pingcap/br/pkg/lightning/web" "github.com/pingcap/br/pkg/lightning/worker" @@ -146,7 +151,7 @@ type RestoreController struct { ioWorkers *worker.Pool checksumWorks *worker.Pool pauser *common.Pauser - backend kv.Backend + backend backend.Backend tidbGlue glue.Glue postProcessLock sync.Mutex // a simple way to ensure post-processing is not concurrent without using complicated goroutines alterTableLock sync.Mutex @@ -207,11 +212,11 @@ func NewRestoreControllerWithPauser( return nil, errors.Trace(err) } - var backend kv.Backend + var backend backend.Backend switch cfg.TikvImporter.Backend { case config.BackendImporter: var err error - backend, err = kv.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) + backend, err = importer.NewImporter(ctx, tls, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr) if err != nil { return nil, errors.Annotate(err, "open importer backend failed") } @@ -220,10 +225,10 @@ func NewRestoreControllerWithPauser( if err != nil { return nil, errors.Annotate(err, "open tidb backend failed") } - backend = kv.NewTiDBBackend(db, cfg.TikvImporter.OnDuplicate) + backend = tidb.NewTiDBBackend(db, cfg.TikvImporter.OnDuplicate) case config.BackendLocal: var rLimit uint64 - rLimit, err = kv.GetSystemRLimit() + rLimit, err = local.GetSystemRLimit() if err != nil { return nil, err } @@ -233,7 +238,7 @@ func NewRestoreControllerWithPauser( maxOpenFiles = math.MaxInt32 } - backend, err = kv.NewLocalBackend(ctx, tls, cfg.TiDB.PdAddr, &cfg.TikvImporter, + backend, err = local.NewLocalBackend(ctx, tls, cfg.TiDB.PdAddr, &cfg.TikvImporter, cfg.Checkpoint.Enable, g, maxOpenFiles) if err != nil { return nil, errors.Annotate(err, "build local backend failed") @@ -685,8 +690,8 @@ func verifyLocalFile(ctx context.Context, cpdb CheckpointsDB, dir string) error } for tableName, engineIDs := range targetTables { for _, engineID := range engineIDs { - _, eID := kv.MakeUUID(tableName, engineID) - file := kv.LocalFile{Uuid: eID} + _, eID := backend.MakeUUID(tableName, engineID) + file := local.File{Uuid: eID} err := file.Exist(dir) if err != nil { log.L().Error("can't find local file", @@ -1280,7 +1285,7 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle // but `indexEngineCp.Status == CheckpointStatusImported` could happen // when kill lightning after saving index engine checkpoint status before saving // table checkpoint status. - var closedIndexEngine *kv.ClosedEngine + var closedIndexEngine *backend.ClosedEngine var restoreErr error // if index-engine checkpoint is lower than `CheckpointStatusClosed`, there must be // data-engines that need to be restore or import. Otherwise, all data-engines should @@ -1410,10 +1415,10 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle func (t *TableRestore) restoreEngine( pCtx context.Context, rc *RestoreController, - indexEngine *kv.OpenedEngine, + indexEngine *backend.OpenedEngine, engineID int32, cp *EngineCheckpoint, -) (*kv.ClosedEngine, error) { +) (*backend.ClosedEngine, error) { ctx, cancel := context.WithCancel(pCtx) defer cancel() // all data has finished written, we can close the engine directly. @@ -1574,7 +1579,7 @@ func (t *TableRestore) restoreEngine( func (t *TableRestore) importEngine( ctx context.Context, - closedEngine *kv.ClosedEngine, + closedEngine *backend.ClosedEngine, rc *RestoreController, engineID int32, cp *EngineCheckpoint, @@ -1743,12 +1748,12 @@ func (rc *RestoreController) fullCompact(ctx context.Context) error { func (rc *RestoreController) doCompact(ctx context.Context, level int32) error { tls := rc.tls.WithHost(rc.cfg.TiDB.PdAddr) - return kv.ForAllStores( + return tikv.ForAllStores( ctx, tls, - kv.StoreStateDisconnected, - func(c context.Context, store *kv.Store) error { - return kv.Compact(c, tls, store.Address, level) + tikv.StoreStateDisconnected, + func(c context.Context, store *tikv.Store) error { + return tikv.Compact(c, tls, store.Address, level) }, ) } @@ -1767,21 +1772,21 @@ func (rc *RestoreController) switchTiKVMode(ctx context.Context, mode sstpb.Swit // since we're running it periodically, so we exclude disconnected stores. // But it is essential all stores be switched back to Normal mode to allow // normal operation. - var minState kv.StoreState + var minState tikv.StoreState if mode == sstpb.SwitchMode_Import { - minState = kv.StoreStateOffline + minState = tikv.StoreStateOffline } else { - minState = kv.StoreStateDisconnected + minState = tikv.StoreStateDisconnected } tls := rc.tls.WithHost(rc.cfg.TiDB.PdAddr) // we ignore switch mode failure since it is not fatal. // no need log the error, it is done in kv.SwitchMode already. - _ = kv.ForAllStores( + _ = tikv.ForAllStores( ctx, tls, minState, - func(c context.Context, store *kv.Store) error { - return kv.SwitchMode(c, tls, store.Address, mode) + func(c context.Context, store *tikv.Store) error { + return tikv.SwitchMode(c, tls, store.Address, mode) }, ) } @@ -2183,7 +2188,7 @@ func getColumnNames(tableInfo *model.TableInfo, permutation []int) []string { func (tr *TableRestore) importKV( ctx context.Context, - closedEngine *kv.ClosedEngine, + closedEngine *backend.ClosedEngine, rc *RestoreController, engineID int32, ) error { @@ -2260,7 +2265,7 @@ func (cr *chunkRestore) deliverLoop( kvsCh <-chan []deliveredKVs, t *TableRestore, engineID int32, - dataEngine, indexEngine *kv.LocalEngineWriter, + dataEngine, indexEngine *backend.LocalEngineWriter, rc *RestoreController, ) (deliverTotalDur time.Duration, err error) { var channelClosed bool @@ -2504,7 +2509,7 @@ func (cr *chunkRestore) restore( ctx context.Context, t *TableRestore, engineID int32, - dataEngine, indexEngine *kv.LocalEngineWriter, + dataEngine, indexEngine *backend.LocalEngineWriter, rc *RestoreController, ) error { // Create the encoder. diff --git a/pkg/lightning/restore/restore_test.go b/pkg/lightning/restore/restore_test.go index 70865a950..e55b3d893 100644 --- a/pkg/lightning/restore/restore_test.go +++ b/pkg/lightning/restore/restore_test.go @@ -35,7 +35,10 @@ import ( "github.com/pingcap/tidb/ddl" tmock "github.com/pingcap/tidb/util/mock" - kv "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/importer" + "github.com/pingcap/br/pkg/lightning/backend/kv" + "github.com/pingcap/br/pkg/lightning/backend/tidb" "github.com/pingcap/br/pkg/lightning/checkpoints" . "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" @@ -723,7 +726,7 @@ func (s *tableRestoreSuite) TestImportKVSuccess(c *C) { controller := gomock.NewController(c) defer controller.Finish() mockBackend := mock.NewMockBackend(controller) - importer := kv.MakeBackend(mockBackend) + importer := backend.MakeBackend(mockBackend) chptCh := make(chan saveCp) defer close(chptCh) rc := &RestoreController{saveCpCh: chptCh} @@ -755,7 +758,7 @@ func (s *tableRestoreSuite) TestImportKVFailure(c *C) { controller := gomock.NewController(c) defer controller.Finish() mockBackend := mock.NewMockBackend(controller) - importer := kv.MakeBackend(mockBackend) + importer := backend.MakeBackend(mockBackend) chptCh := make(chan saveCp) defer close(chptCh) rc := &RestoreController{saveCpCh: chptCh} @@ -784,7 +787,7 @@ func (s *tableRestoreSuite) TestCheckRequirements(c *C) { controller := gomock.NewController(c) defer controller.Finish() mockBackend := mock.NewMockBackend(controller) - backend := kv.MakeBackend(mockBackend) + backend := backend.MakeBackend(mockBackend) ctx := context.Background() @@ -834,7 +837,7 @@ func (s *chunkRestoreSuite) TearDownTest(c *C) { } func (s *chunkRestoreSuite) TestDeliverLoopCancel(c *C) { - rc := &RestoreController{backend: kv.NewMockImporter(nil, "")} + rc := &RestoreController{backend: importer.NewMockImporter(nil, "")} ctx, cancel := context.WithCancel(context.Background()) kvsCh := make(chan []deliveredKVs) @@ -851,7 +854,7 @@ func (s *chunkRestoreSuite) TestDeliverLoopEmptyData(c *C) { controller := gomock.NewController(c) defer controller.Finish() mockBackend := mock.NewMockBackend(controller) - importer := kv.MakeBackend(mockBackend) + importer := backend.MakeBackend(mockBackend) mockBackend.EXPECT().OpenEngine(ctx, gomock.Any()).Return(nil).Times(2) mockBackend.EXPECT().MakeEmptyRows().Return(kv.MakeRowsFromKvPairs(nil)).AnyTimes() @@ -891,7 +894,7 @@ func (s *chunkRestoreSuite) TestDeliverLoop(c *C) { controller := gomock.NewController(c) defer controller.Finish() mockBackend := mock.NewMockBackend(controller) - importer := kv.MakeBackend(mockBackend) + importer := backend.MakeBackend(mockBackend) mockBackend.EXPECT().OpenEngine(ctx, gomock.Any()).Return(nil).Times(2) mockBackend.EXPECT().MakeEmptyRows().Return(kv.MakeRowsFromKvPairs(nil)).AnyTimes() @@ -1085,7 +1088,7 @@ func (s *chunkRestoreSuite) TestEncodeLoopColumnsMismatch(c *C) { kvsCh := make(chan []deliveredKVs, 2) deliverCompleteCh := make(chan deliverResult) - kvEncoder, err := kv.NewTiDBBackend(nil, config.ReplaceOnDup).NewEncoder( + kvEncoder, err := tidb.NewTiDBBackend(nil, config.ReplaceOnDup).NewEncoder( s.tr.encTable, &kv.SessionOptions{ SQLMode: s.cfg.TiDB.SQLMode, @@ -1108,7 +1111,7 @@ func (s *chunkRestoreSuite) TestRestore(c *C) { mockClient := mock.NewMockImportKVClient(controller) mockDataWriter := mock.NewMockImportKV_WriteEngineClient(controller) mockIndexWriter := mock.NewMockImportKV_WriteEngineClient(controller) - importer := kv.NewMockImporter(mockClient, "127.0.0.1:2379") + importer := importer.NewMockImporter(mockClient, "127.0.0.1:2379") mockClient.EXPECT().OpenEngine(ctx, gomock.Any()).Return(nil, nil) mockClient.EXPECT().OpenEngine(ctx, gomock.Any()).Return(nil, nil) @@ -1216,7 +1219,7 @@ func (s *restoreSchemaSuite) SetUpTest(c *C) { FetchRemoteTableModels(gomock.Any(), gomock.Any()). AnyTimes(). Return(make([]*model.TableInfo, 0), nil) - s.rc.backend = kv.MakeBackend(mockBackend) + s.rc.backend = backend.MakeBackend(mockBackend) mockSQLExecutor := mock.NewMockSQLExecutor(s.controller) mockSQLExecutor.EXPECT(). ExecuteWithLog(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). diff --git a/pkg/lightning/backend/tikv.go b/pkg/lightning/tikv/tikv.go similarity index 82% rename from pkg/lightning/backend/tikv.go rename to pkg/lightning/tikv/tikv.go index 006b0066f..a3c37b93d 100644 --- a/pkg/lightning/backend/tikv.go +++ b/pkg/lightning/tikv/tikv.go @@ -11,15 +11,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package backend +package tikv import ( "context" + "fmt" "regexp" + "strings" + "github.com/coreos/go-semver/semver" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/debugpb" "github.com/pingcap/kvproto/pkg/import_sstpb" + "github.com/pingcap/parser/model" "go.uber.org/zap" "golang.org/x/sync/errgroup" "google.golang.org/grpc" @@ -28,6 +32,8 @@ import ( "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/log" + "github.com/pingcap/br/pkg/pdutil" + "github.com/pingcap/br/pkg/version" ) // StoreState is the state of a TiKV store. The numerical value is sorted by @@ -192,3 +198,37 @@ func FetchModeFromMetrics(metrics string) (import_sstpb.SwitchMode, error) { return import_sstpb.SwitchMode_Normal, nil } } + +func FetchRemoteTableModelsFromTLS(ctx context.Context, tls *common.TLS, schema string) ([]*model.TableInfo, error) { + var tables []*model.TableInfo + err := tls.GetJSON(ctx, "/schema/"+schema, &tables) + if err != nil { + return nil, errors.Annotatef(err, "cannot read schema '%s' from remote", schema) + } + return tables, nil +} + +func CheckPDVersion(ctx context.Context, tls *common.TLS, pdAddr string, requiredMinVersion, requiredMaxVersion semver.Version) error { + ver, err := pdutil.FetchPDVersion(ctx, tls, pdAddr) + if err != nil { + return errors.Trace(err) + } + + return version.CheckVersion("PD", *ver, requiredMinVersion, requiredMaxVersion) +} + +func CheckTiKVVersion(ctx context.Context, tls *common.TLS, pdAddr string, requiredMinVersion, requiredMaxVersion semver.Version) error { + return ForAllStores( + ctx, + tls.WithHost(pdAddr), + StoreStateDown, + func(c context.Context, store *Store) error { + component := fmt.Sprintf("TiKV (at %s)", store.Address) + ver, err := semver.NewVersion(strings.TrimPrefix(store.Version, "v")) + if err != nil { + return errors.Annotate(err, component) + } + return version.CheckVersion(component, *ver, requiredMinVersion, requiredMaxVersion) + }, + ) +} diff --git a/pkg/lightning/tikv/tikv_test.go b/pkg/lightning/tikv/tikv_test.go new file mode 100644 index 000000000..72a89f92c --- /dev/null +++ b/pkg/lightning/tikv/tikv_test.go @@ -0,0 +1,251 @@ +package tikv_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "sync" + + "github.com/coreos/go-semver/semver" + . "github.com/pingcap/check" + "github.com/pingcap/kvproto/pkg/import_sstpb" + + "github.com/pingcap/br/pkg/lightning/common" + kv "github.com/pingcap/br/pkg/lightning/tikv" +) + +type tikvSuite struct{} + +var _ = Suite(&tikvSuite{}) + +var ( + // Samples from importer backend for testing the Check***Version functions. + // No need keep these versions in sync. + requiredMinPDVersion = *semver.New("2.1.0") + requiredMinTiKVVersion = *semver.New("2.1.0") + requiredMaxPDVersion = *semver.New("6.0.0") + requiredMaxTiKVVersion = *semver.New("6.0.0") +) + +func (s *tikvSuite) TestForAllStores(c *C) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Write([]byte(` + { + "count": 5, + "stores": [ + { + "store": { + "id": 1, + "address": "127.0.0.1:20160", + "version": "3.0.0-beta.1", + "state_name": "Up" + }, + "status": {} + }, + { + "store": { + "id": 2, + "address": "127.0.0.1:20161", + "version": "3.0.0-rc.1", + "state_name": "Down" + }, + "status": {} + }, + { + "store": { + "id": 3, + "address": "127.0.0.1:20162", + "version": "3.0.0-rc.2", + "state_name": "Disconnected" + }, + "status": {} + }, + { + "store": { + "id": 4, + "address": "127.0.0.1:20163", + "version": "3.0.0", + "state_name": "Tombstone" + }, + "status": {} + }, + { + "store": { + "id": 5, + "address": "127.0.0.1:20164", + "version": "3.0.1", + "state_name": "Offline" + }, + "status": {} + } + ] + } + `)) + })) + defer server.Close() + + ctx := context.Background() + var ( + allStoresLock sync.Mutex + allStores []*kv.Store + ) + tls := common.NewTLSFromMockServer(server) + err := kv.ForAllStores(ctx, tls, kv.StoreStateDown, func(c2 context.Context, store *kv.Store) error { + allStoresLock.Lock() + allStores = append(allStores, store) + allStoresLock.Unlock() + return nil + }) + c.Assert(err, IsNil) + + sort.Slice(allStores, func(i, j int) bool { return allStores[i].Address < allStores[j].Address }) + + c.Assert(allStores, DeepEquals, []*kv.Store{ + { + Address: "127.0.0.1:20160", + Version: "3.0.0-beta.1", + State: kv.StoreStateUp, + }, + { + Address: "127.0.0.1:20161", + Version: "3.0.0-rc.1", + State: kv.StoreStateDown, + }, + { + Address: "127.0.0.1:20162", + Version: "3.0.0-rc.2", + State: kv.StoreStateDisconnected, + }, + { + Address: "127.0.0.1:20164", + Version: "3.0.1", + State: kv.StoreStateOffline, + }, + }) +} + +func (s *tikvSuite) TestFetchModeFromMetrics(c *C) { + testCases := []struct { + metrics string + mode import_sstpb.SwitchMode + isErr bool + }{ + { + metrics: `tikv_config_rocksdb{cf="default",name="hard_pending_compaction_bytes_limit"} 274877906944`, + mode: import_sstpb.SwitchMode_Normal, + }, + { + metrics: `tikv_config_rocksdb{cf="default",name="hard_pending_compaction_bytes_limit"} 0`, + mode: import_sstpb.SwitchMode_Import, + }, + { + metrics: ``, + isErr: true, + }, + } + + for _, tc := range testCases { + comment := Commentf("test case '%s'", tc.metrics) + mode, err := kv.FetchModeFromMetrics(tc.metrics) + if tc.isErr { + c.Assert(err, NotNil, comment) + } else { + c.Assert(err, IsNil, comment) + c.Assert(mode, Equals, tc.mode, comment) + } + } +} + +func (s *tikvSuite) TestCheckPDVersion(c *C) { + var version string + ctx := context.Background() + + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + c.Assert(req.URL.Path, Equals, "/pd/api/v1/version") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(version)) + c.Assert(err, IsNil) + })) + mockURL, err := url.Parse(mockServer.URL) + c.Assert(err, IsNil) + + tls := common.NewTLSFromMockServer(mockServer) + + version = `{ + "version": "v4.0.0-rc.2-451-g760fb650" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), IsNil) + + version = `{ + "version": "v4.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), IsNil) + + version = `{ + "version": "v9999.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") + + version = `{ + "version": "v6.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") + + version = `{ + "version": "v6.0.0-beta" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") + + version = `{ + "version": "v1.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too old.*") +} + +func (s *tikvSuite) TestCheckTiKVVersion(c *C) { + var versions []string + ctx := context.Background() + + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + c.Assert(req.URL.Path, Equals, "/pd/api/v1/stores") + w.WriteHeader(http.StatusOK) + + stores := make([]map[string]interface{}, 0, len(versions)) + for i, v := range versions { + stores = append(stores, map[string]interface{}{ + "store": map[string]interface{}{ + "address": fmt.Sprintf("tikv%d.test:20160", i), + "version": v, + }, + }) + } + err := json.NewEncoder(w).Encode(map[string]interface{}{ + "count": len(versions), + "stores": stores, + }) + c.Assert(err, IsNil) + })) + mockURL, err := url.Parse(mockServer.URL) + c.Assert(err, IsNil) + + tls := common.NewTLSFromMockServer(mockServer) + + versions = []string{"4.1.0", "v4.1.0-alpha-9-ga27a7dd"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), IsNil) + + versions = []string{"9999.0.0", "4.0.0"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) + + versions = []string{"4.0.0", "1.0.0"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv1\.test:20160\) version too old.*`) + + versions = []string{"6.0.0"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) + + versions = []string{"6.0.0-beta"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) +} diff --git a/pkg/mock/backend.go b/pkg/mock/backend.go index 6961c5b83..b579063fd 100644 --- a/pkg/mock/backend.go +++ b/pkg/mock/backend.go @@ -1,52 +1,47 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/pingcap/br/pkg/lightning/backend (interfaces: AbstractBackend,Encoder,Rows,Row,EngineWriter) +// Source: github.com/pingcap/br/pkg/lightning/backend (interfaces: AbstractBackend,EngineWriter) -//+build br_test - -// $ mockgen -package mock -mock_names 'AbstractBackend=MockBackend' github.com/pingcap/br/pkg/lightning/backend AbstractBackend,Encoder,Rows,Row,EngineWriter +// $ mockgen -package mock -mock_names 'AbstractBackend=MockBackend' github.com/pingcap/br/pkg/lightning/backend AbstractBackend,EngineWriter // Package mock is a generated GoMock package. package mock import ( context "context" - reflect "reflect" - time "time" - gomock "github.com/golang/mock/gomock" uuid "github.com/google/uuid" backend "github.com/pingcap/br/pkg/lightning/backend" - log "github.com/pingcap/br/pkg/lightning/log" - verification "github.com/pingcap/br/pkg/lightning/verification" + kv "github.com/pingcap/br/pkg/lightning/backend/kv" model "github.com/pingcap/parser/model" table "github.com/pingcap/tidb/table" - types "github.com/pingcap/tidb/types" + reflect "reflect" + time "time" ) -// MockBackend is a mock of AbstractBackend interface. +// MockBackend is a mock of AbstractBackend interface type MockBackend struct { ctrl *gomock.Controller recorder *MockBackendMockRecorder } -// MockBackendMockRecorder is the mock recorder for MockBackend. +// MockBackendMockRecorder is the mock recorder for MockBackend type MockBackendMockRecorder struct { mock *MockBackend } -// NewMockBackend creates a new mock instance. +// NewMockBackend creates a new mock instance func NewMockBackend(ctrl *gomock.Controller) *MockBackend { mock := &MockBackend{ctrl: ctrl} mock.recorder = &MockBackendMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use. +// EXPECT returns an object that allows the caller to indicate expected use func (m *MockBackend) EXPECT() *MockBackendMockRecorder { return m.recorder } -// CheckRequirements mocks base method. +// CheckRequirements mocks base method func (m *MockBackend) CheckRequirements(arg0 context.Context) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CheckRequirements", arg0) @@ -54,13 +49,13 @@ func (m *MockBackend) CheckRequirements(arg0 context.Context) error { return ret0 } -// CheckRequirements indicates an expected call of CheckRequirements. +// CheckRequirements indicates an expected call of CheckRequirements func (mr *MockBackendMockRecorder) CheckRequirements(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckRequirements", reflect.TypeOf((*MockBackend)(nil).CheckRequirements), arg0) } -// CleanupEngine mocks base method. +// CleanupEngine mocks base method func (m *MockBackend) CleanupEngine(arg0 context.Context, arg1 uuid.UUID) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CleanupEngine", arg0, arg1) @@ -68,25 +63,25 @@ func (m *MockBackend) CleanupEngine(arg0 context.Context, arg1 uuid.UUID) error return ret0 } -// CleanupEngine indicates an expected call of CleanupEngine. +// CleanupEngine indicates an expected call of CleanupEngine func (mr *MockBackendMockRecorder) CleanupEngine(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CleanupEngine", reflect.TypeOf((*MockBackend)(nil).CleanupEngine), arg0, arg1) } -// Close mocks base method. +// Close mocks base method func (m *MockBackend) Close() { m.ctrl.T.Helper() m.ctrl.Call(m, "Close") } -// Close indicates an expected call of Close. +// Close indicates an expected call of Close func (mr *MockBackendMockRecorder) Close() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockBackend)(nil).Close)) } -// CloseEngine mocks base method. +// CloseEngine mocks base method func (m *MockBackend) CloseEngine(arg0 context.Context, arg1 uuid.UUID) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CloseEngine", arg0, arg1) @@ -94,13 +89,13 @@ func (m *MockBackend) CloseEngine(arg0 context.Context, arg1 uuid.UUID) error { return ret0 } -// CloseEngine indicates an expected call of CloseEngine. +// CloseEngine indicates an expected call of CloseEngine func (mr *MockBackendMockRecorder) CloseEngine(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseEngine", reflect.TypeOf((*MockBackend)(nil).CloseEngine), arg0, arg1) } -// EngineFileSizes mocks base method. +// EngineFileSizes mocks base method func (m *MockBackend) EngineFileSizes() []backend.EngineFileSize { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "EngineFileSizes") @@ -108,13 +103,13 @@ func (m *MockBackend) EngineFileSizes() []backend.EngineFileSize { return ret0 } -// EngineFileSizes indicates an expected call of EngineFileSizes. +// EngineFileSizes indicates an expected call of EngineFileSizes func (mr *MockBackendMockRecorder) EngineFileSizes() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EngineFileSizes", reflect.TypeOf((*MockBackend)(nil).EngineFileSizes)) } -// FetchRemoteTableModels mocks base method. +// FetchRemoteTableModels mocks base method func (m *MockBackend) FetchRemoteTableModels(arg0 context.Context, arg1 string) ([]*model.TableInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FetchRemoteTableModels", arg0, arg1) @@ -123,13 +118,13 @@ func (m *MockBackend) FetchRemoteTableModels(arg0 context.Context, arg1 string) return ret0, ret1 } -// FetchRemoteTableModels indicates an expected call of FetchRemoteTableModels. +// FetchRemoteTableModels indicates an expected call of FetchRemoteTableModels func (mr *MockBackendMockRecorder) FetchRemoteTableModels(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchRemoteTableModels", reflect.TypeOf((*MockBackend)(nil).FetchRemoteTableModels), arg0, arg1) } -// FlushAllEngines mocks base method. +// FlushAllEngines mocks base method func (m *MockBackend) FlushAllEngines(arg0 context.Context) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FlushAllEngines", arg0) @@ -137,13 +132,13 @@ func (m *MockBackend) FlushAllEngines(arg0 context.Context) error { return ret0 } -// FlushAllEngines indicates an expected call of FlushAllEngines. +// FlushAllEngines indicates an expected call of FlushAllEngines func (mr *MockBackendMockRecorder) FlushAllEngines(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushAllEngines", reflect.TypeOf((*MockBackend)(nil).FlushAllEngines), arg0) } -// FlushEngine mocks base method. +// FlushEngine mocks base method func (m *MockBackend) FlushEngine(arg0 context.Context, arg1 uuid.UUID) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FlushEngine", arg0, arg1) @@ -151,13 +146,13 @@ func (m *MockBackend) FlushEngine(arg0 context.Context, arg1 uuid.UUID) error { return ret0 } -// FlushEngine indicates an expected call of FlushEngine. +// FlushEngine indicates an expected call of FlushEngine func (mr *MockBackendMockRecorder) FlushEngine(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlushEngine", reflect.TypeOf((*MockBackend)(nil).FlushEngine), arg0, arg1) } -// ImportEngine mocks base method. +// ImportEngine mocks base method func (m *MockBackend) ImportEngine(arg0 context.Context, arg1 uuid.UUID) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ImportEngine", arg0, arg1) @@ -165,7 +160,7 @@ func (m *MockBackend) ImportEngine(arg0 context.Context, arg1 uuid.UUID) error { return ret0 } -// ImportEngine indicates an expected call of ImportEngine. +// ImportEngine indicates an expected call of ImportEngine func (mr *MockBackendMockRecorder) ImportEngine(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImportEngine", reflect.TypeOf((*MockBackend)(nil).ImportEngine), arg0, arg1) @@ -186,36 +181,36 @@ func (mr *MockBackendMockRecorder) LocalWriter(arg0, arg1 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LocalWriter", reflect.TypeOf((*MockBackend)(nil).LocalWriter), arg0, arg1) } -// MakeEmptyRows mocks base method. -func (m *MockBackend) MakeEmptyRows() backend.Rows { +// MakeEmptyRows mocks base method +func (m *MockBackend) MakeEmptyRows() kv.Rows { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "MakeEmptyRows") - ret0, _ := ret[0].(backend.Rows) + ret0, _ := ret[0].(kv.Rows) return ret0 } -// MakeEmptyRows indicates an expected call of MakeEmptyRows. +// MakeEmptyRows indicates an expected call of MakeEmptyRows func (mr *MockBackendMockRecorder) MakeEmptyRows() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeEmptyRows", reflect.TypeOf((*MockBackend)(nil).MakeEmptyRows)) } -// NewEncoder mocks base method. -func (m *MockBackend) NewEncoder(arg0 table.Table, arg1 *backend.SessionOptions) (backend.Encoder, error) { +// NewEncoder mocks base method +func (m *MockBackend) NewEncoder(arg0 table.Table, arg1 *kv.SessionOptions) (kv.Encoder, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NewEncoder", arg0, arg1) - ret0, _ := ret[0].(backend.Encoder) + ret0, _ := ret[0].(kv.Encoder) ret1, _ := ret[1].(error) return ret0, ret1 } -// NewEncoder indicates an expected call of NewEncoder. +// NewEncoder indicates an expected call of NewEncoder func (mr *MockBackendMockRecorder) NewEncoder(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewEncoder", reflect.TypeOf((*MockBackend)(nil).NewEncoder), arg0, arg1) } -// OpenEngine mocks base method. +// OpenEngine mocks base method func (m *MockBackend) OpenEngine(arg0 context.Context, arg1 uuid.UUID) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "OpenEngine", arg0, arg1) @@ -223,13 +218,13 @@ func (m *MockBackend) OpenEngine(arg0 context.Context, arg1 uuid.UUID) error { return ret0 } -// OpenEngine indicates an expected call of OpenEngine. +// OpenEngine indicates an expected call of OpenEngine func (mr *MockBackendMockRecorder) OpenEngine(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OpenEngine", reflect.TypeOf((*MockBackend)(nil).OpenEngine), arg0, arg1) } -// ResetEngine mocks base method. +// ResetEngine mocks base method func (m *MockBackend) ResetEngine(arg0 context.Context, arg1 uuid.UUID) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ResetEngine", arg0, arg1) @@ -237,13 +232,13 @@ func (m *MockBackend) ResetEngine(arg0 context.Context, arg1 uuid.UUID) error { return ret0 } -// ResetEngine indicates an expected call of ResetEngine. +// ResetEngine indicates an expected call of ResetEngine func (mr *MockBackendMockRecorder) ResetEngine(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetEngine", reflect.TypeOf((*MockBackend)(nil).ResetEngine), arg0, arg1) } -// RetryImportDelay mocks base method. +// RetryImportDelay mocks base method func (m *MockBackend) RetryImportDelay() time.Duration { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RetryImportDelay") @@ -251,13 +246,13 @@ func (m *MockBackend) RetryImportDelay() time.Duration { return ret0 } -// RetryImportDelay indicates an expected call of RetryImportDelay. +// RetryImportDelay indicates an expected call of RetryImportDelay func (mr *MockBackendMockRecorder) RetryImportDelay() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RetryImportDelay", reflect.TypeOf((*MockBackend)(nil).RetryImportDelay)) } -// ShouldPostProcess mocks base method. +// ShouldPostProcess mocks base method func (m *MockBackend) ShouldPostProcess() bool { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ShouldPostProcess") @@ -265,186 +260,50 @@ func (m *MockBackend) ShouldPostProcess() bool { return ret0 } -// ShouldPostProcess indicates an expected call of ShouldPostProcess. +// ShouldPostProcess indicates an expected call of ShouldPostProcess func (mr *MockBackendMockRecorder) ShouldPostProcess() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShouldPostProcess", reflect.TypeOf((*MockBackend)(nil).ShouldPostProcess)) } -// MockEncoder is a mock of Encoder interface. -type MockEncoder struct { - ctrl *gomock.Controller - recorder *MockEncoderMockRecorder -} - -// MockEncoderMockRecorder is the mock recorder for MockEncoder. -type MockEncoderMockRecorder struct { - mock *MockEncoder -} - -// NewMockEncoder creates a new mock instance. -func NewMockEncoder(ctrl *gomock.Controller) *MockEncoder { - mock := &MockEncoder{ctrl: ctrl} - mock.recorder = &MockEncoderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockEncoder) EXPECT() *MockEncoderMockRecorder { - return m.recorder -} - -// Close mocks base method. -func (m *MockEncoder) Close() { - m.ctrl.T.Helper() - m.ctrl.Call(m, "Close") -} - -// Close indicates an expected call of Close. -func (mr *MockEncoderMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockEncoder)(nil).Close)) -} - -// Encode mocks base method. -func (m *MockEncoder) Encode(arg0 log.Logger, arg1 []types.Datum, arg2 int64, arg3 []int) (backend.Row, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Encode", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(backend.Row) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Encode indicates an expected call of Encode. -func (mr *MockEncoderMockRecorder) Encode(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Encode", reflect.TypeOf((*MockEncoder)(nil).Encode), arg0, arg1, arg2, arg3) -} - -// MockRows is a mock of Rows interface. -type MockRows struct { - ctrl *gomock.Controller - recorder *MockRowsMockRecorder -} - -// MockRowsMockRecorder is the mock recorder for MockRows. -type MockRowsMockRecorder struct { - mock *MockRows -} - -// NewMockRows creates a new mock instance. -func NewMockRows(ctrl *gomock.Controller) *MockRows { - mock := &MockRows{ctrl: ctrl} - mock.recorder = &MockRowsMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRows) EXPECT() *MockRowsMockRecorder { - return m.recorder -} - -// Clear mocks base method. -func (m *MockRows) Clear() backend.Rows { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Clear") - ret0, _ := ret[0].(backend.Rows) - return ret0 -} - -// Clear indicates an expected call of Clear. -func (mr *MockRowsMockRecorder) Clear() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clear", reflect.TypeOf((*MockRows)(nil).Clear)) -} - -// SplitIntoChunks mocks base method. -func (m *MockRows) SplitIntoChunks(arg0 int) []backend.Rows { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SplitIntoChunks", arg0) - ret0, _ := ret[0].([]backend.Rows) - return ret0 -} - -// SplitIntoChunks indicates an expected call of SplitIntoChunks. -func (mr *MockRowsMockRecorder) SplitIntoChunks(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SplitIntoChunks", reflect.TypeOf((*MockRows)(nil).SplitIntoChunks), arg0) -} - -// MockRow is a mock of Row interface. -type MockRow struct { - ctrl *gomock.Controller - recorder *MockRowMockRecorder -} - -// MockRowMockRecorder is the mock recorder for MockRow. -type MockRowMockRecorder struct { - mock *MockRow -} - -// NewMockRow creates a new mock instance. -func NewMockRow(ctrl *gomock.Controller) *MockRow { - mock := &MockRow{ctrl: ctrl} - mock.recorder = &MockRowMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRow) EXPECT() *MockRowMockRecorder { - return m.recorder -} - -// ClassifyAndAppend mocks base method. -func (m *MockRow) ClassifyAndAppend(arg0 *backend.Rows, arg1 *verification.KVChecksum, arg2 *backend.Rows, arg3 *verification.KVChecksum) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ClassifyAndAppend", arg0, arg1, arg2, arg3) -} - -// ClassifyAndAppend indicates an expected call of ClassifyAndAppend. -func (mr *MockRowMockRecorder) ClassifyAndAppend(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClassifyAndAppend", reflect.TypeOf((*MockRow)(nil).ClassifyAndAppend), arg0, arg1, arg2, arg3) -} - -// MockEngineWriter is a mock of EngineWriter interface. +// MockEngineWriter is a mock of EngineWriter interface type MockEngineWriter struct { ctrl *gomock.Controller recorder *MockEngineWriterMockRecorder } -// MockEngineWriterMockRecorder is the mock recorder for MockEngineWriter. +// MockEngineWriterMockRecorder is the mock recorder for MockEngineWriter type MockEngineWriterMockRecorder struct { mock *MockEngineWriter } -// NewMockEngineWriter creates a new mock instance. +// NewMockEngineWriter creates a new mock instance func NewMockEngineWriter(ctrl *gomock.Controller) *MockEngineWriter { mock := &MockEngineWriter{ctrl: ctrl} mock.recorder = &MockEngineWriterMockRecorder{mock} return mock } -// EXPECT returns an object that allows the caller to indicate expected use. +// EXPECT returns an object that allows the caller to indicate expected use func (m *MockEngineWriter) EXPECT() *MockEngineWriterMockRecorder { return m.recorder } -// AppendRows mocks base method. -func (m *MockEngineWriter) AppendRows(arg0 context.Context, arg1 string, arg2 []string, arg3 uint64, arg4 backend.Rows) error { +// AppendRows mocks base method +func (m *MockEngineWriter) AppendRows(arg0 context.Context, arg1 string, arg2 []string, arg3 uint64, arg4 kv.Rows) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AppendRows", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(error) return ret0 } -// AppendRows indicates an expected call of AppendRows. +// AppendRows indicates an expected call of AppendRows func (mr *MockEngineWriterMockRecorder) AppendRows(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRows", reflect.TypeOf((*MockEngineWriter)(nil).AppendRows), arg0, arg1, arg2, arg3, arg4) } -// Close mocks base method. +// Close mocks base method func (m *MockEngineWriter) Close() error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Close") @@ -452,7 +311,7 @@ func (m *MockEngineWriter) Close() error { return ret0 } -// Close indicates an expected call of Close. +// Close indicates an expected call of Close func (mr *MockEngineWriterMockRecorder) Close() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockEngineWriter)(nil).Close)) diff --git a/pkg/mock/dummy.go b/pkg/mock/dummy.go deleted file mode 100644 index 21b43d21e..000000000 --- a/pkg/mock/dummy.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2021 PingCAP, Inc. Licensed under Apache-2.0. - -// This file avoids mock being an empty package when `-tags br_test` is not provided. - -package mock diff --git a/pkg/mock/glue.go b/pkg/mock/glue.go index 2c5f74bf5..cd4446a88 100644 --- a/pkg/mock/glue.go +++ b/pkg/mock/glue.go @@ -1,8 +1,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/pingcap/br/pkg/lightning/glue/glue.go -//+build br_test - // Package mock is a generated GoMock package. package mock diff --git a/pkg/mock/glue_checkpoint.go b/pkg/mock/glue_checkpoint.go index 73df5c4e8..50bf3fc36 100644 --- a/pkg/mock/glue_checkpoint.go +++ b/pkg/mock/glue_checkpoint.go @@ -1,8 +1,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/pingcap/br/pkg/lightning/checkpoints/glue_checkpoint.go -//+build br_test - // Package mock is a generated GoMock package. package mock diff --git a/pkg/mock/importer.go b/pkg/mock/importer.go index 50c9611f8..a58c0d05b 100644 --- a/pkg/mock/importer.go +++ b/pkg/mock/importer.go @@ -1,8 +1,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/pingcap/kvproto/pkg/import_kvpb (interfaces: ImportKVClient,ImportKV_WriteEngineClient) -//+build br_test - // $ mockgen -package mock github.com/pingcap/kvproto/pkg/import_kvpb ImportKVClient,ImportKV_WriteEngineClient // Package mock is a generated GoMock package. diff --git a/pkg/mock/kv.go b/pkg/mock/kv.go new file mode 100644 index 000000000..a285cd0d5 --- /dev/null +++ b/pkg/mock/kv.go @@ -0,0 +1,152 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/pingcap/br/pkg/lightning/backend/kv (interfaces: Encoder,Rows,Row) + +// $ mockgen -package mock github.com/pingcap/br/pkg/lightning/backend/kv Encoder,Rows,Row + +// Package mock is a generated GoMock package. +package mock + +import ( + gomock "github.com/golang/mock/gomock" + kv "github.com/pingcap/br/pkg/lightning/backend/kv" + log "github.com/pingcap/br/pkg/lightning/log" + verification "github.com/pingcap/br/pkg/lightning/verification" + types "github.com/pingcap/tidb/types" + reflect "reflect" +) + +// MockEncoder is a mock of Encoder interface +type MockEncoder struct { + ctrl *gomock.Controller + recorder *MockEncoderMockRecorder +} + +// MockEncoderMockRecorder is the mock recorder for MockEncoder +type MockEncoderMockRecorder struct { + mock *MockEncoder +} + +// NewMockEncoder creates a new mock instance +func NewMockEncoder(ctrl *gomock.Controller) *MockEncoder { + mock := &MockEncoder{ctrl: ctrl} + mock.recorder = &MockEncoderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockEncoder) EXPECT() *MockEncoderMockRecorder { + return m.recorder +} + +// Close mocks base method +func (m *MockEncoder) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close +func (mr *MockEncoderMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockEncoder)(nil).Close)) +} + +// Encode mocks base method +func (m *MockEncoder) Encode(arg0 log.Logger, arg1 []types.Datum, arg2 int64, arg3 []int) (kv.Row, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Encode", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(kv.Row) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Encode indicates an expected call of Encode +func (mr *MockEncoderMockRecorder) Encode(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Encode", reflect.TypeOf((*MockEncoder)(nil).Encode), arg0, arg1, arg2, arg3) +} + +// MockRows is a mock of Rows interface +type MockRows struct { + ctrl *gomock.Controller + recorder *MockRowsMockRecorder +} + +// MockRowsMockRecorder is the mock recorder for MockRows +type MockRowsMockRecorder struct { + mock *MockRows +} + +// NewMockRows creates a new mock instance +func NewMockRows(ctrl *gomock.Controller) *MockRows { + mock := &MockRows{ctrl: ctrl} + mock.recorder = &MockRowsMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockRows) EXPECT() *MockRowsMockRecorder { + return m.recorder +} + +// Clear mocks base method +func (m *MockRows) Clear() kv.Rows { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Clear") + ret0, _ := ret[0].(kv.Rows) + return ret0 +} + +// Clear indicates an expected call of Clear +func (mr *MockRowsMockRecorder) Clear() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Clear", reflect.TypeOf((*MockRows)(nil).Clear)) +} + +// SplitIntoChunks mocks base method +func (m *MockRows) SplitIntoChunks(arg0 int) []kv.Rows { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SplitIntoChunks", arg0) + ret0, _ := ret[0].([]kv.Rows) + return ret0 +} + +// SplitIntoChunks indicates an expected call of SplitIntoChunks +func (mr *MockRowsMockRecorder) SplitIntoChunks(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SplitIntoChunks", reflect.TypeOf((*MockRows)(nil).SplitIntoChunks), arg0) +} + +// MockRow is a mock of Row interface +type MockRow struct { + ctrl *gomock.Controller + recorder *MockRowMockRecorder +} + +// MockRowMockRecorder is the mock recorder for MockRow +type MockRowMockRecorder struct { + mock *MockRow +} + +// NewMockRow creates a new mock instance +func NewMockRow(ctrl *gomock.Controller) *MockRow { + mock := &MockRow{ctrl: ctrl} + mock.recorder = &MockRowMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockRow) EXPECT() *MockRowMockRecorder { + return m.recorder +} + +// ClassifyAndAppend mocks base method +func (m *MockRow) ClassifyAndAppend(arg0 *kv.Rows, arg1 *verification.KVChecksum, arg2 *kv.Rows, arg3 *verification.KVChecksum) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ClassifyAndAppend", arg0, arg1, arg2, arg3) +} + +// ClassifyAndAppend indicates an expected call of ClassifyAndAppend +func (mr *MockRowMockRecorder) ClassifyAndAppend(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClassifyAndAppend", reflect.TypeOf((*MockRow)(nil).ClassifyAndAppend), arg0, arg1, arg2, arg3) +} diff --git a/pkg/mock/mock_cluster.go b/pkg/mock/mock_cluster.go index 85ee700ac..8c24ca300 100644 --- a/pkg/mock/mock_cluster.go +++ b/pkg/mock/mock_cluster.go @@ -1,7 +1,5 @@ // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. -//+build br_test - package mock import ( diff --git a/pkg/mock/mock_cluster_test.go b/pkg/mock/mock_cluster_test.go index 9e5455383..2ca00923c 100644 --- a/pkg/mock/mock_cluster_test.go +++ b/pkg/mock/mock_cluster_test.go @@ -1,7 +1,5 @@ // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. -//+build br_test - package mock_test import ( diff --git a/pkg/mock/s3iface.go b/pkg/mock/s3iface.go index 3b65420a0..e8151f124 100644 --- a/pkg/mock/s3iface.go +++ b/pkg/mock/s3iface.go @@ -1,8 +1,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/aws/aws-sdk-go/service/s3/s3iface (interfaces: S3API) -//+build br_test - // $ mockgen -package mock github.com/aws/aws-sdk-go/service/s3/s3iface S3API > pkg/mock/s3iface.go // Package mock is a generated GoMock package. diff --git a/pkg/mock/storage.go b/pkg/mock/storage.go index e64cc3ed1..d029ef08b 100644 --- a/pkg/mock/storage.go +++ b/pkg/mock/storage.go @@ -1,8 +1,6 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/pingcap/br/pkg/storage (interfaces: ExternalStorage) -//+build br_test - // Package mock is a generated GoMock package. package mock diff --git a/pkg/version/version.go b/pkg/version/version.go index 9bc11c173..5f1ce4eaf 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -151,12 +151,15 @@ func CheckVersion(component string, actual, requiredMinVersion, requiredMaxVersi actual, ) } - if actual.Compare(requiredMaxVersion) >= 0 { + // Compare the major version number to make sure beta version does not pass + // the check. This is because beta version may contains incompatible + // changes. + if actual.Major >= requiredMaxVersion.Major { return errors.Annotatef(berrors.ErrVersionMismatch, - "%s version too new, expected to be within [%s, %s), found '%s'", + "%s version too new, expected to be within [%s, %d.0.0), found '%s'", component, requiredMinVersion, - requiredMaxVersion, + requiredMaxVersion.Major, actual, ) } @@ -187,3 +190,12 @@ func ExtractTiDBVersion(version string) (*semver.Version, error) { rawVersion = strings.TrimPrefix(rawVersion, "v") return semver.NewVersion(rawVersion) } + +// CheckTiDBVersion is equals to ExtractTiDBVersion followed by CheckVersion. +func CheckTiDBVersion(versionStr string, requiredMinVersion, requiredMaxVersion semver.Version) error { + version, err := ExtractTiDBVersion(versionStr) + if err != nil { + return errors.Trace(err) + } + return CheckVersion("TiDB", *version, requiredMinVersion, requiredMaxVersion) +} diff --git a/pkg/version/version_test.go b/pkg/version/version_test.go index f3c4747a3..7e9e344a3 100644 --- a/pkg/version/version_test.go +++ b/pkg/version/version_test.go @@ -215,3 +215,17 @@ func (s *checkSuite) TestExtractTiDBVersion(c *C) { _, err = ExtractTiDBVersion("not-a-valid-version") c.Assert(err, NotNil) } + +func (s *checkSuite) TestCheckVersion(c *C) { + err := CheckVersion("TiNB", *semver.New("2.3.5"), *semver.New("2.1.0"), *semver.New("3.0.0")) + c.Assert(err, IsNil) + + err = CheckVersion("TiNB", *semver.New("2.1.0"), *semver.New("2.3.5"), *semver.New("3.0.0")) + c.Assert(err, ErrorMatches, "TiNB version too old.*") + + err = CheckVersion("TiNB", *semver.New("3.1.0"), *semver.New("2.3.5"), *semver.New("3.0.0")) + c.Assert(err, ErrorMatches, "TiNB version too new.*") + + err = CheckVersion("TiNB", *semver.New("3.0.0-beta"), *semver.New("2.3.5"), *semver.New("3.0.0")) + c.Assert(err, ErrorMatches, "TiNB version too new.*") +} diff --git a/tests/README.md b/tests/README.md index 5030a8f7b..db039886c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,15 +18,13 @@ You can also run unit tests directly via `go test` like: ```sh make failpoint-enable -go test -tags br_test github.com/pingcap/br/pkg/cdclog --test.v --check.v --check.f TestColumn -# ^~~~~~~~~~~~~ +go test github.com/pingcap/br/pkg/cdclog --test.v --check.v --check.f TestColumn make failpoint-disable ``` but note that: -* the build-tag `br_test` must be enabled (this workarounds the lack of test-dependencies in go.mod) * failpoints must be toggled manually # Integration tests diff --git a/tests/br_log_restore/run.sh b/tests/br_log_restore/run.sh index d101479a3..d03403dba 100755 --- a/tests/br_log_restore/run.sh +++ b/tests/br_log_restore/run.sh @@ -105,7 +105,7 @@ run_sql "DROP DATABASE ${DB}_DDL1" run_sql "DROP DATABASE ${DB}_DDL2" # restore full -export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/FailIngestMeta=return("notleader")' +export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/local/FailIngestMeta=return("notleader")' echo "restore start..." run_br restore cdclog -s "s3://$BUCKET/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \ --log-file "restore.log" --log-level "info" --start-ts $start_ts --end-ts $end_ts @@ -129,7 +129,7 @@ if [ "$row_count" -ne "0" ]; then echo "TEST: [$TEST_NAME] fail on ts range test." fi -export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/FailIngestMeta=return("epochnotmatch")' +export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/local/FailIngestMeta=return("epochnotmatch")' echo "restore again to restore a=5 record..." run_br restore cdclog -s "s3://$BUCKET/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \ --log-file "restore.log" --log-level "info" --start-ts $end_ts diff --git a/tests/lightning_local_backend/run.sh b/tests/lightning_local_backend/run.sh index 4e7b38b19..f822debd4 100755 --- a/tests/lightning_local_backend/run.sh +++ b/tests/lightning_local_backend/run.sh @@ -23,7 +23,7 @@ ENGINE_COUNT=6 rm -f "$TEST_DIR/lightning-local.log" rm -f "/tmp/tidb_lightning_checkpoint_local_backend_test.pb" run_sql 'DROP DATABASE IF EXISTS cpeng;' -export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/FailIngestMeta=1*return("notleader")' +export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/local/FailIngestMeta=1*return("notleader")' run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" @@ -40,7 +40,7 @@ check_contains 'sum(c): 46' run_sql 'DROP DATABASE cpeng;' rm -f "/tmp/tidb_lightning_checkpoint_local_backend_test.pb" -export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/FailIngestMeta=2*return("epochnotmatch")' +export GO_FAILPOINTS='github.com/pingcap/br/pkg/lightning/backend/local/FailIngestMeta=2*return("epochnotmatch")' run_lightning --backend local --enable-checkpoint=1 --log-file "$TEST_DIR/lightning-local.log" --config "tests/$TEST_NAME/config.toml" diff --git a/tests/lightning_tidb_duplicate_data/run.sh b/tests/lightning_tidb_duplicate_data/run.sh index 3a4d41a44..46a964a41 100644 --- a/tests/lightning_tidb_duplicate_data/run.sh +++ b/tests/lightning_tidb_duplicate_data/run.sh @@ -22,7 +22,7 @@ sed -i.bak 's/new/old/g' "tests/lightning_tidb_duplicate_data/data/dup.dup.sql" for type in replace ignore error; do run_sql 'DROP DATABASE IF EXISTS dup;' - export GO_FAILPOINTS="github.com/pingcap/br/pkg/lightning/backend/FailIfImportedSomeRows=return" + export GO_FAILPOINTS="github.com/pingcap/br/pkg/lightning/backend/tidb/FailIfImportedSomeRows=return" set +e run_lightning --config "tests/$TEST_NAME/$type.toml" 2> /dev/null ERRORCODE=$?