From 317cd1c32b9265ef85d1b956da192f0fcd9b2a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=B1=E5=B2=9A?= <36239017+YuJuncen@users.noreply.github.com> Date: Wed, 7 Apr 2021 12:50:32 +0800 Subject: [PATCH] cherry pick #766 to release-4.0 Signed-off-by: ti-srebot --- Makefile | 6 +- cmd/tidb-lightning-ctl/main.go | 12 +- cmd/tidb-lightning-ctl/main_test.go | 2 +- cmd/tidb-lightning/main.go | 11 +- cmd/tidb-lightning/main_test.go | 2 +- pkg/lightning/backend/backend_test.go | 3 +- pkg/lightning/backend/importer.go | 14 +- pkg/lightning/backend/local.go | 193 +++++-- pkg/lightning/backend/local_test.go | 39 +- pkg/lightning/backend/localhelper_test.go | 109 +++- pkg/lightning/backend/session.go | 17 +- pkg/lightning/backend/sql2kv.go | 55 +- pkg/lightning/backend/sql2kv_test.go | 9 +- pkg/lightning/backend/tidb.go | 53 +- pkg/lightning/backend/tidb_test.go | 5 +- pkg/lightning/checkpoints/checkpoints.go | 69 ++- .../checkpoints/checkpoints_file_test.go | 2 - pkg/lightning/checkpoints/glue_checkpoint.go | 44 +- pkg/lightning/common/pause_test.go | 6 +- pkg/lightning/common/security_test.go | 6 +- pkg/lightning/common/util.go | 2 +- pkg/lightning/common/util_test.go | 4 +- pkg/lightning/config/config.go | 322 ++++++----- pkg/lightning/config/configlist.go | 24 +- pkg/lightning/glue/glue.go | 4 +- pkg/lightning/lightning.go | 43 +- pkg/lightning/lightning_test.go | 34 +- pkg/lightning/log/log.go | 4 +- pkg/lightning/metric/metric.go | 1 + pkg/lightning/mock/glue.go | 4 +- pkg/lightning/mydump/csv_parser.go | 15 +- pkg/lightning/mydump/loader.go | 51 +- pkg/lightning/mydump/loader_test.go | 7 +- pkg/lightning/mydump/parser.go | 11 +- pkg/lightning/mydump/reader_test.go | 50 +- pkg/lightning/mydump/region.go | 10 +- pkg/lightning/mydump/region_test.go | 15 - pkg/lightning/mydump/router.go | 1 - pkg/lightning/mydump/router_test.go | 2 +- pkg/lightning/restore/checksum.go | 40 +- pkg/lightning/restore/checksum_test.go | 16 +- pkg/lightning/restore/const.go | 18 - pkg/lightning/restore/restore.go | 500 ++++++++++++------ pkg/lightning/restore/restore_test.go | 277 +++++++--- pkg/lightning/restore/tidb.go | 12 +- pkg/lightning/restore/tidb_test.go | 8 +- pkg/lightning/tikv/tikv_test.go | 252 +++++++++ pkg/lightning/web/progress.go | 5 +- 48 files changed, 1638 insertions(+), 751 deletions(-) delete mode 100644 pkg/lightning/restore/const.go create mode 100644 pkg/lightning/tikv/tikv_test.go diff --git a/Makefile b/Makefile index ec9ed0a1a..2ee37e572 100644 --- a/Makefile +++ b/Makefile @@ -181,6 +181,8 @@ static: prepare tools @# exhaustivestruct - Protobuf structs have hidden fields, like "XXX_NoUnkeyedLiteral" @# exhaustive - no need to check exhaustiveness of enum switch statements @# gosec - too many false positive + @# errorlint - pingcap/errors is incompatible with std errors. + @# wrapcheck - there are too many unwrapped errors in tidb-lightning CGO_ENABLED=0 tools/bin/golangci-lint run --enable-all --deadline 120s \ --disable gochecknoglobals \ --disable goimports \ @@ -201,7 +203,9 @@ static: prepare tools --disable exhaustive \ --disable godot \ --disable gosec \ - $$($(PACKAGE_DIRECTORIES) | grep -v "lightning") + --disable errorlint \ + --disable wrapcheck \ + $(PACKAGE_DIRECTORIES) # pingcap/errors APIs are mixed with multiple patterns 'pkg/errors', # 'juju/errors' and 'pingcap/parser'. To avoid confusion and mistake, # we only allow a subset of APIs, that's "Normalize|Annotate|Trace|Cause". diff --git a/cmd/tidb-lightning-ctl/main.go b/cmd/tidb-lightning-ctl/main.go index 54e04a312..ca4af5486 100644 --- a/cmd/tidb-lightning-ctl/main.go +++ b/cmd/tidb-lightning-ctl/main.go @@ -58,7 +58,7 @@ func run() error { // there is a check if `-d` points to a valid storage, and '' is not. // since tidb-lightning-ctl does not need `-d` we change the default to a valid but harmless value. dFlag := fs.Lookup("d") - dFlag.Value.Set("noop://") + _ = dFlag.Value.Set("noop://") dFlag.DefValue = "noop://" compact = fs.Bool("compact", false, "do manual compaction on the target cluster") @@ -244,8 +244,8 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common if err != nil { fmt.Fprintln(os.Stderr, "* Encountered error while closing engine:", err) lastErr = err - } else { - closedEngine.Cleanup(ctx) + } else if err := closedEngine.Cleanup(ctx); err != nil { + lastErr = err } } } @@ -259,8 +259,13 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common for _, table := range targetTables { for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ { fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID) +<<<<<<< HEAD _, eID := kv.MakeUUID(table.TableName, engineID) file := kv.LocalFile{Uuid: eID} +======= + _, eID := backend.MakeUUID(table.TableName, engineID) + file := local.File{UUID: eID} +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) err := file.Cleanup(cfg.TikvImporter.SortedKVDir) if err != nil { fmt.Fprintln(os.Stderr, "* Encountered error while cleanup engine:", err) @@ -318,6 +323,7 @@ func checkpointDump(ctx context.Context, cfg *config.Config, dumpFolder string) } func getLocalStoringTables(ctx context.Context, cfg *config.Config) (err2 error) { + //nolint:prealloc // This is a placeholder. var tables []string defer func() { if err2 == nil { diff --git a/cmd/tidb-lightning-ctl/main_test.go b/cmd/tidb-lightning-ctl/main_test.go index bef107137..e0351c20e 100644 --- a/cmd/tidb-lightning-ctl/main_test.go +++ b/cmd/tidb-lightning-ctl/main_test.go @@ -19,7 +19,7 @@ import ( "testing" ) -func TestRunMain(t *testing.T) { +func TestRunMain(_ *testing.T) { if _, isIntegrationTest := os.LookupEnv("INTEGRATION_TEST"); !isIntegrationTest { // override exit to pass unit test. exit = func(code int) {} diff --git a/cmd/tidb-lightning/main.go b/cmd/tidb-lightning/main.go index 7bb40f734..c5f967ba0 100644 --- a/cmd/tidb-lightning/main.go +++ b/cmd/tidb-lightning/main.go @@ -78,13 +78,12 @@ func main() { err = func() error { if globalCfg.App.ServerMode { return app.RunServer() - } else { - cfg := config.NewConfig() - if err := cfg.LoadFromGlobal(globalCfg); err != nil { - return err - } - return app.RunOnce(context.Background(), cfg, nil) } + cfg := config.NewConfig() + if err := cfg.LoadFromGlobal(globalCfg); err != nil { + return err + } + return app.RunOnce(context.Background(), cfg, nil) }() if err != nil { diff --git a/cmd/tidb-lightning/main_test.go b/cmd/tidb-lightning/main_test.go index 26ee27d45..c77e38a88 100644 --- a/cmd/tidb-lightning/main_test.go +++ b/cmd/tidb-lightning/main_test.go @@ -21,7 +21,7 @@ import ( "testing" ) -func TestRunMain(t *testing.T) { +func TestRunMain(_ *testing.T) { if _, isIntegrationTest := os.LookupEnv("INTEGRATION_TEST"); !isIntegrationTest { // override exit to pass unit test. exit = func(code int) {} diff --git a/pkg/lightning/backend/backend_test.go b/pkg/lightning/backend/backend_test.go index d32a31646..a6be981d7 100644 --- a/pkg/lightning/backend/backend_test.go +++ b/pkg/lightning/backend/backend_test.go @@ -27,7 +27,7 @@ var _ = Suite(&backendSuite{}) // FIXME: Cannot use the real SetUpTest/TearDownTest to set up the mock // otherwise the mock error will be ignored. -func (s *backendSuite) setUpTest(c *C) { +func (s *backendSuite) setUpTest(c gomock.TestReporter) { s.controller = gomock.NewController(c) s.mockBackend = mock.NewMockBackend(s.controller) s.backend = kv.MakeBackend(s.mockBackend) @@ -292,6 +292,7 @@ func (s *backendSuite) TestImportFailedRecovered(c *C) { c.Assert(err, IsNil) } +//nolint:interfacer // change test case signature causes check panicking. func (s *backendSuite) TestClose(c *C) { s.setUpTest(c) defer s.tearDownTest() diff --git a/pkg/lightning/backend/importer.go b/pkg/lightning/backend/importer.go index d63cc3ef1..67285e50a 100644 --- a/pkg/lightning/backend/importer.go +++ b/pkg/lightning/backend/importer.go @@ -197,6 +197,7 @@ outside: func (importer *importer) WriteRowsToImporter( ctx context.Context, + //nolint:interfacer // false positive engineUUID uuid.UUID, ts uint64, rows Rows, @@ -392,19 +393,28 @@ func (importer *importer) ResetEngine(context.Context, uuid.UUID) error { return errors.New("cannot reset an engine in importer backend") } +<<<<<<< HEAD:pkg/lightning/backend/importer.go func (importer *importer) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (EngineWriter, error) { return &ImporterWriter{importer: importer, engineUUID: engineUUID}, nil +======= +func (importer *importer) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (backend.EngineWriter, error) { + return &Writer{importer: importer, engineUUID: engineUUID}, nil +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/importer/importer.go } -type ImporterWriter struct { +type Writer struct { importer *importer engineUUID uuid.UUID } -func (w *ImporterWriter) Close() error { +func (w *Writer) Close() error { return nil } +<<<<<<< HEAD:pkg/lightning/backend/importer.go func (w *ImporterWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows Rows) error { +======= +func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows kv.Rows) error { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/importer/importer.go return w.importer.WriteRows(ctx, w.engineUUID, tableName, columnNames, ts, rows) } diff --git a/pkg/lightning/backend/local.go b/pkg/lightning/backend/local.go index 6b979f38a..365fe83cb 100644 --- a/pkg/lightning/backend/local.go +++ b/pkg/lightning/backend/local.go @@ -113,15 +113,14 @@ var ( // Range record start and end key for localStoreDir.DB // so we can write it to tikv in streaming type Range struct { - start []byte - end []byte - length int + start []byte + end []byte } // localFileMeta contains some field that is necessary to continue the engine restore/import process. // These field should be written to disk when we update chunk checkpoint type localFileMeta struct { - Ts uint64 `json:"ts"` + TS uint64 `json:"ts"` // Length is the number of KV pairs stored by the engine. Length atomic.Int64 `json:"length"` // TotalSize is the total pre-compressed KV byte size stored by engine. @@ -140,7 +139,7 @@ const ( type LocalFile struct { localFileMeta db *pebble.DB - Uuid uuid.UUID + UUID uuid.UUID localWriters sync.Map // isImportingAtomic is an atomic variable indicating whether the importMutex has been locked. @@ -149,8 +148,13 @@ type LocalFile struct { mutex sync.Mutex } +<<<<<<< HEAD:pkg/lightning/backend/local.go func (e *LocalFile) Close() error { log.L().Debug("closing local engine", zap.Stringer("engine", e.Uuid), zap.Stack("stack")) +======= +func (e *File) Close() error { + log.L().Debug("closing local engine", zap.Stringer("engine", e.UUID), zap.Stack("stack")) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go if e.db == nil { return nil } @@ -160,14 +164,24 @@ func (e *LocalFile) Close() error { } // Cleanup remove meta and db files +<<<<<<< HEAD:pkg/lightning/backend/local.go func (e *LocalFile) Cleanup(dataDir string) error { dbPath := filepath.Join(dataDir, e.Uuid.String()) +======= +func (e *File) Cleanup(dataDir string) error { + dbPath := filepath.Join(dataDir, e.UUID.String()) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go return os.RemoveAll(dbPath) } // Exist checks if db folder existing (meta sometimes won't flush before lightning exit) +<<<<<<< HEAD:pkg/lightning/backend/local.go func (e *LocalFile) Exist(dataDir string) error { dbPath := filepath.Join(dataDir, e.Uuid.String()) +======= +func (e *File) Exist(dataDir string) error { + dbPath := filepath.Join(dataDir, e.UUID.String()) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go if _, err := os.Stat(dbPath); err != nil { return err } @@ -177,7 +191,7 @@ func (e *LocalFile) Exist(dataDir string) error { func (e *LocalFile) getSizeProperties() (*sizeProperties, error) { sstables, err := e.db.SSTables(pebble.WithProperties()) if err != nil { - log.L().Warn("get table properties failed", zap.Stringer("engine", e.Uuid), log.ShortError(err)) + log.L().Warn("get table properties failed", zap.Stringer("engine", e.UUID), log.ShortError(err)) return nil, errors.Trace(err) } @@ -188,7 +202,7 @@ func (e *LocalFile) getSizeProperties() (*sizeProperties, error) { data := hack.Slice(prop) rangeProps, err := decodeRangeProperties(data) if err != nil { - log.L().Warn("decodeRangeProperties failed", zap.Stringer("engine", e.Uuid), + log.L().Warn("decodeRangeProperties failed", zap.Stringer("engine", e.UUID), zap.Stringer("fileNum", info.FileNum), log.ShortError(err)) return nil, errors.Trace(err) } @@ -210,7 +224,7 @@ func (e *LocalFile) getEngineFileSize() EngineFileSize { total := metrics.Total() var memSize int64 e.localWriters.Range(func(k, v interface{}) bool { - w := k.(*LocalWriter) + w := k.(*Writer) memSize += w.writeBatch.totalSize if w.writer != nil { total.Size += int64(w.writer.writer.EstimatedSize()) @@ -218,8 +232,13 @@ func (e *LocalFile) getEngineFileSize() EngineFileSize { return true }) +<<<<<<< HEAD:pkg/lightning/backend/local.go return EngineFileSize{ UUID: e.Uuid, +======= + return backend.EngineFileSize{ + UUID: e.UUID, +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go DiskSize: total.Size, MemSize: memSize, IsImporting: e.isLocked(), @@ -255,7 +274,7 @@ func (e *LocalFile) flushLocalWriters(parentCtx context.Context) error { eg, ctx := errgroup.WithContext(parentCtx) e.localWriters.Range(func(k, v interface{}) bool { eg.Go(func() error { - w := k.(*LocalWriter) + w := k.(*Writer) replyErrCh := make(chan error, 1) w.flushChMutex.RLock() if w.flushCh != nil { @@ -309,14 +328,14 @@ func (e *LocalFile) saveEngineMeta() error { func (e *LocalFile) loadEngineMeta() { jsonBytes, closer, err := e.db.Get(engineMetaKey) if err != nil { - log.L().Debug("local db missing engine meta", zap.Stringer("uuid", e.Uuid), zap.Error(err)) + log.L().Debug("local db missing engine meta", zap.Stringer("uuid", e.UUID), zap.Error(err)) return } defer closer.Close() err = json.Unmarshal(jsonBytes, &e.localFileMeta) if err != nil { - log.L().Warn("local db failed to deserialize meta", zap.Stringer("uuid", e.Uuid), zap.ByteString("content", jsonBytes), zap.Error(err)) + log.L().Warn("local db failed to deserialize meta", zap.Stringer("uuid", e.UUID), zap.ByteString("content", jsonBytes), zap.Error(err)) } } @@ -365,7 +384,6 @@ type connPool struct { mu sync.Mutex conns []*grpc.ClientConn - name string next int cap int newConn func(ctx context.Context) (*grpc.ClientConn, error) @@ -478,10 +496,17 @@ func NewLocalBackend( return MakeBackend(local), nil } +<<<<<<< HEAD:pkg/lightning/backend/local.go // lock locks a local file and returns the LocalFile instance if it exists. func (local *local) lockEngine(engineId uuid.UUID, state importMutexState) *LocalFile { if e, ok := local.engines.Load(engineId); ok { engine := e.(*LocalFile) +======= +// lock locks a local file and returns the File instance if it exists. +func (local *local) lockEngine(engineID uuid.UUID, state importMutexState) *File { + if e, ok := local.engines.Load(engineID); ok { + engine := e.(*File) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go engine.lock(state) return engine } @@ -569,13 +594,13 @@ func (local *local) Close() { } // FlushEngine ensure the written data is saved successfully, to make sure no data lose after restart -func (local *local) FlushEngine(ctx context.Context, engineId uuid.UUID) error { - engineFile := local.lockEngine(engineId, importMutexStateFlush) +func (local *local) FlushEngine(ctx context.Context, engineID uuid.UUID) error { + engineFile := local.lockEngine(engineID, importMutexStateFlush) // the engine cannot be deleted after while we've acquired the lock identified by UUID. if engineFile == nil { - return errors.Errorf("engine '%s' not found", engineId) + return errors.Errorf("engine '%s' not found", engineID) } defer engineFile.unlock() return engineFile.flushEngineWithoutLock(ctx) @@ -639,8 +664,13 @@ func (local *local) OpenEngine(ctx context.Context, engineUUID uuid.UUID) error if err != nil { return err } +<<<<<<< HEAD:pkg/lightning/backend/local.go e, _ := local.engines.LoadOrStore(engineUUID, &LocalFile{Uuid: engineUUID}) engine := e.(*LocalFile) +======= + e, _ := local.engines.LoadOrStore(engineUUID, &File{UUID: engineUUID}) + engine := e.(*File) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go engine.db = db engine.loadEngineMeta() return nil @@ -663,8 +693,13 @@ func (local *local) CloseEngine(ctx context.Context, engineUUID uuid.UUID) error } return err } +<<<<<<< HEAD:pkg/lightning/backend/local.go engineFile := &LocalFile{ Uuid: engineUUID, +======= + engineFile := &File{ + UUID: engineUUID, +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go db: db, } engineFile.loadEngineMeta() @@ -764,7 +799,7 @@ func (local *local) WriteToTiKV( } req.Chunk = &sst.WriteRequest_Batch{ Batch: &sst.WriteBatch{ - CommitTs: engineFile.Ts, + CommitTs: engineFile.TS, }, } clients = append(clients, wstream) @@ -829,11 +864,9 @@ func (local *local) WriteToTiKV( for i, wStream := range clients { if resp, closeErr := wStream.CloseAndRecv(); closeErr != nil { return nil, nil, stats, closeErr - } else { - if leaderID == region.Region.Peers[i].GetId() { - leaderPeerMetas = resp.Metas - log.L().Debug("get metas after write kv stream to tikv", zap.Reflect("metas", leaderPeerMetas)) - } + } else if leaderID == region.Region.Peers[i].GetId() { + leaderPeerMetas = resp.Metas + log.L().Debug("get metas after write kv stream to tikv", zap.Reflect("metas", leaderPeerMetas)) } } @@ -949,7 +982,7 @@ func (local *local) readAndSplitIntoRange(engineFile *LocalFile) ([]Range, error // <= 96MB no need to split into range if engineFileTotalSize <= local.regionSplitSize && engineFileLength <= regionMaxKeyCount { - ranges := []Range{{start: firstKey, end: endKey, length: int(engineFileLength)}} + ranges := []Range{{start: firstKey, end: endKey}} return ranges, nil } @@ -961,7 +994,7 @@ func (local *local) readAndSplitIntoRange(engineFile *LocalFile) ([]Range, error ranges := splitRangeBySizeProps(Range{start: firstKey, end: endKey}, sizeProps, local.regionSplitSize, regionMaxKeyCount*2/3) - log.L().Info("split engine key ranges", zap.Stringer("engine", engineFile.Uuid), + log.L().Info("split engine key ranges", zap.Stringer("engine", engineFile.UUID), zap.Int64("totalSize", engineFileTotalSize), zap.Int64("totalCount", engineFileLength), log.ZapRedactBinary("firstKey", firstKey), log.ZapRedactBinary("lastKey", lastKey), zap.Int("ranges", len(ranges))) @@ -978,12 +1011,8 @@ type bytesRecycleChan struct { // NOTE: we don't used a `sync.Pool` because when will sync.Pool release is depending on the // garbage collector which always release the memory so late. Use a fixed size chan to reuse // can decrease the memory usage to 1/3 compare with sync.Pool. -var recycleChan *bytesRecycleChan - -func init() { - recycleChan = &bytesRecycleChan{ - ch: make(chan []byte, 1024), - } +var recycleChan = &bytesRecycleChan{ + ch: make(chan []byte, 1024), } func (c *bytesRecycleChan) Acquire() []byte { @@ -1018,7 +1047,7 @@ func newBytesBuffer() *bytesBuffer { func (b *bytesBuffer) addBuf() { if b.curBufIdx < len(b.bufs)-1 { - b.curBufIdx += 1 + b.curBufIdx++ b.curBuf = b.bufs[b.curBufIdx] } else { buf := recycleChan.Acquire() @@ -1269,7 +1298,7 @@ loopWrite: func (local *local) writeAndIngestByRanges(ctx context.Context, engineFile *LocalFile, ranges []Range, remainRanges *syncdRanges) error { if engineFile.Length.Load() == 0 { // engine is empty, this is likes because it's a index engine but the table contains no index - log.L().Info("engine contains no data", zap.Stringer("uuid", engineFile.Uuid)) + log.L().Info("engine contains no data", zap.Stringer("uuid", engineFile.UUID)) return nil } log.L().Debug("the ranges Length write to tikv", zap.Int("Length", len(ranges))) @@ -1460,6 +1489,76 @@ func (local *local) CheckRequirements(ctx context.Context) error { if err := checkTiKVVersion(ctx, local.tls, local.pdAddr, localMinTiKVVersion, localMaxTiKVVersion); err != nil { return err } +<<<<<<< HEAD:pkg/lightning/backend/local.go +======= + + tidbVersion, _ := version.ExtractTiDBVersion(versionStr) + + return checkTiFlashVersion(ctx, local.g, checkCtx, *tidbVersion) +} + +func checkTiDBVersion(_ context.Context, versionStr string, requiredMinVersion, requiredMaxVersion semver.Version) error { + return version.CheckTiDBVersion(versionStr, requiredMinVersion, requiredMaxVersion) +} + +var tiFlashReplicaQuery = "SELECT TABLE_SCHEMA, TABLE_NAME FROM information_schema.TIFLASH_REPLICA WHERE REPLICA_COUNT > 0;" + +type tblName struct { + schema string + name string +} + +type tblNames []tblName + +func (t tblNames) String() string { + var b strings.Builder + b.WriteByte('[') + for i, n := range t { + if i > 0 { + b.WriteString(", ") + } + b.WriteString(common.UniqueTable(n.schema, n.name)) + } + b.WriteByte(']') + return b.String() +} + +// check TiFlash replicas. +// local backend doesn't support TiFlash before tidb v4.0.5 +func checkTiFlashVersion(ctx context.Context, g glue.Glue, checkCtx *backend.CheckCtx, tidbVersion semver.Version) error { + if tidbVersion.Compare(tiFlashMinVersion) >= 0 { + return nil + } + + res, err := g.GetSQLExecutor().QueryStringsWithLog(ctx, tiFlashReplicaQuery, "fetch tiflash replica info", log.L()) + if err != nil { + return errors.Annotate(err, "fetch tiflash replica info failed") + } + + tiFlashTablesMap := make(map[tblName]struct{}, len(res)) + for _, tblInfo := range res { + name := tblName{schema: tblInfo[0], name: tblInfo[1]} + tiFlashTablesMap[name] = struct{}{} + } + + tiFlashTables := make(tblNames, 0) + for _, dbMeta := range checkCtx.DBMetas { + for _, tblMeta := range dbMeta.Tables { + if len(tblMeta.DataFiles) == 0 { + continue + } + name := tblName{schema: tblMeta.DB, name: tblMeta.Name} + if _, ok := tiFlashTablesMap[name]; ok { + tiFlashTables = append(tiFlashTables, name) + } + } + } + + if len(tiFlashTables) > 0 { + helpInfo := "Please either upgrade TiDB to version >= 4.0.5 or add TiFlash replica after load data." + return errors.Errorf("lightning local backend doesn't support TiFlash in this TiDB version. conflict tables: %s. "+helpInfo, tiFlashTables) + } +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go return nil } @@ -1484,8 +1583,13 @@ func (local *local) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (Engi return openLocalWriter(engineFile, local.localStoreDir, local.localWriterMemCacheSize), nil } +<<<<<<< HEAD:pkg/lightning/backend/local.go func openLocalWriter(f *LocalFile, sstDir string, memtableSizeLimit int64) *LocalWriter { w := &LocalWriter{ +======= +func openLocalWriter(f *File, sstDir string, memtableSizeLimit int64) *Writer { + w := &Writer{ +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go sstDir: sstDir, kvsChan: make(chan []common.KvPair, defaultLocalWriterKVsChannelCap), flushCh: make(chan chan error), @@ -1603,9 +1707,15 @@ func nextKey(key []byte) []byte { // in tikv <= 4.x, tikv will truncate the row key, so we should fetch the next valid row key // See: https://github.com/tikv/tikv/blob/f7f22f70e1585d7ca38a59ea30e774949160c3e8/components/raftstore/src/coprocessor/split_observer.rs#L36-L41 +<<<<<<< HEAD:pkg/lightning/backend/local.go if isRecordKey(key) { tableId, handle, _ := tablecodec.DecodeRecordKey(key) return tablecodec.EncodeRowKeyWithHandle(tableId, handle+1) +======= + if tablecodec.IsRecordKey(key) { + tableID, handle, _ := tablecodec.DecodeRecordKey(key) + return tablecodec.EncodeRowKeyWithHandle(tableID, handle.Next()) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go } // if key is an index, directly append a 0x00 to the key. @@ -1718,7 +1828,7 @@ func (c *RangePropertiesCollector) insertNewPoint(key []byte) { // implement `TablePropertyCollector.Add` func (c *RangePropertiesCollector) Add(key pebble.InternalKey, value []byte) error { c.currentOffsets.Size += uint64(len(value)) + uint64(len(key.UserKey)) - c.currentOffsets.Keys += 1 + c.currentOffsets.Keys++ if len(c.lastKey) == 0 || c.sizeInLastRange() >= c.propSizeIdxDistance || c.keysInLastRange() >= c.propKeysIdxDistance { c.insertNewPoint(key.UserKey) @@ -1789,7 +1899,7 @@ func (local *local) EngineFileSizes() (res []EngineFileSize) { return } -type LocalWriter struct { +type Writer struct { writeErr common.OnceError local *LocalFile consumeCh chan struct{} @@ -1802,9 +1912,14 @@ type LocalWriter struct { writer *sstWriter } +<<<<<<< HEAD:pkg/lightning/backend/local.go // TODO: temporarily replace this async append rows with the former write-batch approach before addressing the performance issue. func (w *LocalWriter) AppendRowsAsync(ctx context.Context, tableName string, columnNames []string, ts uint64, rows Rows) error { kvs := rows.(kvPairs) +======= +func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows kv.Rows) error { + kvs := kv.KvPairsFromRows(rows) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go if len(kvs) == 0 { return nil } @@ -1812,10 +1927,11 @@ func (w *LocalWriter) AppendRowsAsync(ctx context.Context, tableName string, col return err } w.kvsChan <- kvs - w.local.Ts = ts + w.local.TS = ts return nil } +<<<<<<< HEAD:pkg/lightning/backend/local.go // TODO: replace the implementation back with `AppendRowsAsync` after addressing the performance issue. func (w *LocalWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows Rows) error { kvs := rows.(kvPairs) @@ -1840,6 +1956,9 @@ func (w *LocalWriter) AppendRows(ctx context.Context, tableName string, columnNa } func (w *LocalWriter) Close() error { +======= +func (w *Writer) Close() error { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local.go w.local.localWriters.Delete(w) close(w.kvsChan) @@ -1861,11 +1980,11 @@ func (w *LocalWriter) Close() error { } } -func (w *LocalWriter) genSSTPath() string { +func (w *Writer) genSSTPath() string { return filepath.Join(w.sstDir, uuid.New().String()+".sst") } -func (w *LocalWriter) writeRowsLoop() { +func (w *Writer) writeRowsLoop() { defer func() { if w.writer != nil { w.writer.cleanUp() @@ -1930,7 +2049,7 @@ outside: } } -func (w *LocalWriter) writeKVsOrIngest(desc localIngestDescription) error { +func (w *Writer) writeKVsOrIngest(desc localIngestDescription) error { if w.writer != nil { if err := w.writer.writeKVs(&w.writeBatch); err != errorUnorderedSSTInsertion { return err diff --git a/pkg/lightning/backend/local_test.go b/pkg/lightning/backend/local_test.go index 34fbf78d3..5b1166597 100644 --- a/pkg/lightning/backend/local_test.go +++ b/pkg/lightning/backend/local_test.go @@ -63,9 +63,38 @@ func (s *localSuite) TestNextKey(c *C) { // test recode key // key with int handle +<<<<<<< HEAD:pkg/lightning/backend/local_test.go for _, handleId := range []int64{1, 255, math.MaxInt32} { key := tablecodec.EncodeRowKeyWithHandle(1, handleId) c.Assert(nextKey(key), DeepEquals, []byte(tablecodec.EncodeRowKeyWithHandle(1, handleId+1))) +======= + for _, handleID := range []int64{1, 255, math.MaxInt32} { + key := tablecodec.EncodeRowKeyWithHandle(1, tidbkv.IntHandle(handleID)) + c.Assert(nextKey(key), DeepEquals, []byte(tablecodec.EncodeRowKeyWithHandle(1, tidbkv.IntHandle(handleID+1)))) + } + + testDatums := [][]types.Datum{ + {types.NewIntDatum(1), types.NewIntDatum(2)}, + {types.NewIntDatum(255), types.NewIntDatum(256)}, + {types.NewIntDatum(math.MaxInt32), types.NewIntDatum(math.MaxInt32 + 1)}, + {types.NewStringDatum("test"), types.NewStringDatum("test\000")}, + {types.NewStringDatum("test\255"), types.NewStringDatum("test\255\000")}, + } + + stmtCtx := new(stmtctx.StatementContext) + for _, datums := range testDatums { + keyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewIntDatum(123), datums[0]) + c.Assert(err, IsNil) + h, err := tidbkv.NewCommonHandle(keyBytes) + c.Assert(err, IsNil) + key := tablecodec.EncodeRowKeyWithHandle(1, h) + nextKeyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewIntDatum(123), datums[1]) + c.Assert(err, IsNil) + nextHdl, err := tidbkv.NewCommonHandle(nextKeyBytes) + c.Assert(err, IsNil) + expectNextKey := []byte(tablecodec.EncodeRowKeyWithHandle(1, nextHdl)) + c.Assert(nextKey(key), DeepEquals, expectNextKey) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local_test.go } // dIAAAAAAAAD/PV9pgAAAAAD/AAABA4AAAAD/AAAAAQOAAAD/AAAAAAEAAAD8 @@ -285,8 +314,13 @@ func testLocalWriter(c *C, needSort bool, partitialSort bool) { err = os.Mkdir(tmpPath, 0o755) c.Assert(err, IsNil) meta := localFileMeta{} +<<<<<<< HEAD:pkg/lightning/backend/local_test.go _, engineUUID := MakeUUID("ww", 0) f := LocalFile{localFileMeta: meta, db: db, Uuid: engineUUID} +======= + _, engineUUID := backend.MakeUUID("ww", 0) + f := File{localFileMeta: meta, db: db, UUID: engineUUID} +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/local_test.go w := openLocalWriter(&f, tmpPath, 1024*1024) ctx := context.Background() @@ -439,11 +473,12 @@ func (s *localSuite) TestIsIngestRetryable(c *C) { c.Assert(err, NotNil) resp.Error = &errorpb.Error{Message: "raft: proposal dropped"} - retryType, newRegion, err = local.isIngestRetryable(ctx, resp, region, meta) + retryType, _, err = local.isIngestRetryable(ctx, resp, region, meta) c.Assert(retryType, Equals, retryWrite) + c.Assert(err, NotNil) resp.Error = &errorpb.Error{Message: "unknown error"} - retryType, newRegion, err = local.isIngestRetryable(ctx, resp, region, meta) + retryType, _, err = local.isIngestRetryable(ctx, resp, region, meta) c.Assert(retryType, Equals, retryNone) c.Assert(err, ErrorMatches, "non-retryable error: unknown error") } diff --git a/pkg/lightning/backend/localhelper_test.go b/pkg/lightning/backend/localhelper_test.go index 563ee036e..af6f771b7 100644 --- a/pkg/lightning/backend/localhelper_test.go +++ b/pkg/lightning/backend/localhelper_test.go @@ -355,7 +355,7 @@ func (h *noopHook) AfterScanRegions(res []*restore.RegionInfo, err error) ([]*re return res, err } -func (s *localSuite) doTestBatchSplitRegionByRanges(c *C, ctx context.Context, hook clientHook, errPat string) { +func (s *localSuite) doTestBatchSplitRegionByRanges(ctx context.Context, c *C, hook clientHook, errPat string) { oldLimit := maxBatchSplitKeys oldSplitBackoffTime := splitRegionBaseBackOffTime maxBatchSplitKeys = 4 @@ -422,7 +422,7 @@ func (s *localSuite) doTestBatchSplitRegionByRanges(c *C, ctx context.Context, h } func (s *localSuite) TestBatchSplitRegionByRanges(c *C) { - s.doTestBatchSplitRegionByRanges(c, context.Background(), nil, "") + s.doTestBatchSplitRegionByRanges(context.Background(), c, nil, "") } type scanRegionEmptyHook struct { @@ -440,7 +440,7 @@ func (h *scanRegionEmptyHook) AfterScanRegions(res []*restore.RegionInfo, err er } func (s *localSuite) TestBatchSplitRegionByRangesScanFailed(c *C) { - s.doTestBatchSplitRegionByRanges(c, context.Background(), &scanRegionEmptyHook{}, "paginate scan region returns empty result") + s.doTestBatchSplitRegionByRanges(context.Background(), c, &scanRegionEmptyHook{}, "paginate scan region returns empty result") } type splitRegionEpochNotMatchHook struct { @@ -456,7 +456,7 @@ func (h *splitRegionEpochNotMatchHook) BeforeSplitRegion(ctx context.Context, re } func (s *localSuite) TestBatchSplitByRangesEpochNotMatch(c *C) { - s.doTestBatchSplitRegionByRanges(c, context.Background(), &splitRegionEpochNotMatchHook{}, "batch split regions failed: epoch not match.*") + s.doTestBatchSplitRegionByRanges(context.Background(), c, &splitRegionEpochNotMatchHook{}, "batch split regions failed: epoch not match.*") } // return epoch not match error in every other call @@ -478,7 +478,7 @@ func (h *splitRegionEpochNotMatchHookRandom) BeforeSplitRegion(ctx context.Conte } func (s *localSuite) TestBatchSplitByRangesEpochNotMatchOnce(c *C) { - s.doTestBatchSplitRegionByRanges(c, context.Background(), &splitRegionEpochNotMatchHookRandom{}, "") + s.doTestBatchSplitRegionByRanges(context.Background(), c, &splitRegionEpochNotMatchHookRandom{}, "") } type splitRegionNoValidKeyHook struct { @@ -497,11 +497,11 @@ func (h splitRegionNoValidKeyHook) BeforeSplitRegion(ctx context.Context, region } func (s *localSuite) TestBatchSplitByRangesNoValidKeysOnce(c *C) { - s.doTestBatchSplitRegionByRanges(c, context.Background(), &splitRegionNoValidKeyHook{returnErrTimes: 1}, ".*no valid key.*") + s.doTestBatchSplitRegionByRanges(context.Background(), c, &splitRegionNoValidKeyHook{returnErrTimes: 1}, ".*no valid key.*") } func (s *localSuite) TestBatchSplitByRangesNoValidKeys(c *C) { - s.doTestBatchSplitRegionByRanges(c, context.Background(), &splitRegionNoValidKeyHook{returnErrTimes: math.MaxInt32}, ".*no valid key.*") + s.doTestBatchSplitRegionByRanges(context.Background(), c, &splitRegionNoValidKeyHook{returnErrTimes: math.MaxInt32}, ".*no valid key.*") } type reportAfterSplitHook struct { @@ -528,25 +528,112 @@ func (s *localSuite) TestBatchSplitByRangeCtxCanceled(c *C) { } }() - s.doTestBatchSplitRegionByRanges(c, ctx, &reportAfterSplitHook{ch: ch}, ".*context canceled.*") + s.doTestBatchSplitRegionByRanges(ctx, c, &reportAfterSplitHook{ch: ch}, ".*context canceled.*") close(ch) } +<<<<<<< HEAD:pkg/lightning/backend/localhelper_test.go +======= +func (s *localSuite) doTestBatchSplitByRangesWithClusteredIndex(c *C, hook clientHook) { + oldLimit := maxBatchSplitKeys + oldSplitBackoffTime := splitRegionBaseBackOffTime + maxBatchSplitKeys = 10 + splitRegionBaseBackOffTime = time.Millisecond + defer func() { + maxBatchSplitKeys = oldLimit + splitRegionBaseBackOffTime = oldSplitBackoffTime + }() + + stmtCtx := new(stmtctx.StatementContext) + + tableID := int64(1) + tableStartKey := tablecodec.EncodeTablePrefix(tableID) + tableEndKey := tablecodec.EncodeTablePrefix(tableID + 1) + keys := [][]byte{[]byte(""), tableStartKey} + // pre split 2 regions + for i := int64(0); i < 2; i++ { + keyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewIntDatum(i)) + c.Assert(err, IsNil) + h, err := kv.NewCommonHandle(keyBytes) + c.Assert(err, IsNil) + key := tablecodec.EncodeRowKeyWithHandle(tableID, h) + keys = append(keys, key) + } + keys = append(keys, tableEndKey, []byte("")) + client := initTestClient(keys, hook) + local := &local{ + splitCli: client, + } + ctx := context.Background() + + // we batch generate a batch of row keys for table 1 with common handle + rangeKeys := make([][]byte, 0, 20+1) + for i := int64(0); i < 2; i++ { + for j := int64(0); j < 10; j++ { + keyBytes, err := codec.EncodeKey(stmtCtx, nil, types.NewIntDatum(i), types.NewIntDatum(j*10000)) + c.Assert(err, IsNil) + h, err := kv.NewCommonHandle(keyBytes) + c.Assert(err, IsNil) + key := tablecodec.EncodeRowKeyWithHandle(tableID, h) + rangeKeys = append(rangeKeys, key) + } + } + + start := rangeKeys[0] + ranges := make([]Range, 0, len(rangeKeys)-1) + for _, e := range rangeKeys[1:] { + ranges = append(ranges, Range{start: start, end: e}) + start = e + } + + err := local.SplitAndScatterRegionByRanges(ctx, ranges, true) + c.Assert(err, IsNil) + + startKey := codec.EncodeBytes([]byte{}, rangeKeys[0]) + endKey := codec.EncodeBytes([]byte{}, rangeKeys[len(rangeKeys)-1]) + // check split ranges + regions, err := paginateScanRegion(ctx, client, startKey, endKey, 5) + c.Assert(err, IsNil) + c.Assert(len(regions), Equals, len(ranges)+1) + + checkKeys := append([][]byte{}, rangeKeys[:10]...) + checkKeys = append(checkKeys, keys[3]) + checkKeys = append(checkKeys, rangeKeys[10:]...) + checkRegionRanges(c, regions, checkKeys) +} + +func (s *localSuite) TestBatchSplitByRangesWithClusteredIndex(c *C) { + s.doTestBatchSplitByRangesWithClusteredIndex(c, nil) +} + +func (s *localSuite) TestBatchSplitByRangesWithClusteredIndexEpochNotMatch(c *C) { + s.doTestBatchSplitByRangesWithClusteredIndex(c, &splitRegionEpochNotMatchHookRandom{}) +} + +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/localhelper_test.go func (s *localSuite) TestNeedSplit(c *C) { - tableId := int64(1) + tableID := int64(1) peers := make([]*metapb.Peer, 1) peers[0] = &metapb.Peer{ Id: 1, StoreId: 1, } keys := []int64{10, 100, 500, 1000, 999999, -1} +<<<<<<< HEAD:pkg/lightning/backend/localhelper_test.go start := tablecodec.EncodeRowKeyWithHandle(tableId, 0) +======= + start := tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(0)) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/localhelper_test.go regionStart := codec.EncodeBytes([]byte{}, start) regions := make([]*restore.RegionInfo, 0) for _, end := range keys { var regionEndKey []byte if end >= 0 { +<<<<<<< HEAD:pkg/lightning/backend/localhelper_test.go endKey := tablecodec.EncodeRowKeyWithHandle(tableId, end) +======= + endKey := tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(end)) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/localhelper_test.go regionEndKey = codec.EncodeBytes([]byte{}, endKey) } region := &restore.RegionInfo{ @@ -574,7 +661,11 @@ func (s *localSuite) TestNeedSplit(c *C) { } for hdl, idx := range checkMap { +<<<<<<< HEAD:pkg/lightning/backend/localhelper_test.go checkKey := tablecodec.EncodeRowKeyWithHandle(tableId, hdl) +======= + checkKey := tablecodec.EncodeRowKeyWithHandle(tableID, kv.IntHandle(hdl)) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/local/localhelper_test.go res := needSplit(checkKey, regions) if idx < 0 { c.Assert(res, IsNil) diff --git a/pkg/lightning/backend/session.go b/pkg/lightning/backend/session.go index b1cef956e..ec475ecac 100644 --- a/pkg/lightning/backend/session.go +++ b/pkg/lightning/backend/session.go @@ -25,6 +25,9 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/br/pkg/lightning/common" + "github.com/pingcap/br/pkg/lightning/log" + + "go.uber.org/zap" ) // invalidIterator is a trimmed down Iterator type which is invalid. @@ -88,12 +91,15 @@ type transaction struct { kvMemBuf } +<<<<<<< HEAD:pkg/lightning/backend/session.go var _ kv.Transaction = &transaction{} func (t *transaction) NewStagingBuffer() kv.MemBuffer { return &t.kvMemBuf } +======= +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/kv/session.go func (t *transaction) GetMemBuffer() kv.MemBuffer { return &t.kvMemBuf } @@ -175,11 +181,18 @@ func newSession(options *SessionOptions) *session { vars.SQLMode = sqlMode if options.SysVars != nil { for k, v := range options.SysVars { - vars.SetSystemVar(k, v) + if err := vars.SetSystemVar(k, v); err != nil { + log.L().DPanic("new session: failed to set system var", + log.ShortError(err), + zap.String("key", k)) + } } } vars.StmtCtx.TimeZone = vars.Location() - vars.SetSystemVar("timestamp", strconv.FormatInt(options.Timestamp, 10)) + if err := vars.SetSystemVar("timestamp", strconv.FormatInt(options.Timestamp, 10)); err != nil { + log.L().Warn("new session: failed to set timestamp", + log.ShortError(err)) + } vars.TxnCtx = nil s := &session{ diff --git a/pkg/lightning/backend/sql2kv.go b/pkg/lightning/backend/sql2kv.go index 72278c47b..574487709 100644 --- a/pkg/lightning/backend/sql2kv.go +++ b/pkg/lightning/backend/sql2kv.go @@ -91,7 +91,7 @@ func autoRandomIncrementBits(col *table.Column, randomBits int) int { incrementalBits := typeBitsLength - randomBits hasSignBit := !mysql.HasUnsignedFlag(col.Flag) if hasSignBit { - incrementalBits -= 1 + incrementalBits-- } return incrementalBits } @@ -208,11 +208,13 @@ func (row rowArrayMarshaler) MarshalLogArray(encoder zapcore.ArrayEncoder) error return err } } - encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { + if err := encoder.AppendObject(zapcore.ObjectMarshalerFunc(func(enc zapcore.ObjectEncoder) error { enc.AddString("kind", kindStr[kind]) enc.AddString("val", log.RedactString(str)) return nil - })) + })); err != nil { + return err + } } return nil } @@ -271,6 +273,17 @@ func MakeRowFromKvPairs(pairs []common.KvPair) Row { return kvPairs(pairs) } +<<<<<<< HEAD:pkg/lightning/backend/sql2kv.go +======= +// KvPairsFromRows converts a Rows instance constructed from MakeRowsFromKvPairs +// back into a slice of KvPair. This method panics if the Rows is not +// constructed in such way. +// nolint:golint // kv.KvPairsFromRows sounds good. +func KvPairsFromRows(rows Rows) []common.KvPair { + return []common.KvPair(rows.(kvPairs)) +} + +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/kv/sql2kv.go // Encode a row of data into KV pairs. // // See comments in `(*TableRestore).initializeColumns` for the meaning of the @@ -285,6 +298,7 @@ func (kvcodec *tableKVEncoder) Encode( var value types.Datum var err error + //nolint:prealloc // This is a placeholder. var record []types.Datum if kvcodec.recordCache != nil { @@ -298,15 +312,16 @@ func (kvcodec *tableKVEncoder) Encode( j := columnPermutation[i] isAutoIncCol := mysql.HasAutoIncrementFlag(col.Flag) isPk := mysql.HasPriKeyFlag(col.Flag) - if j >= 0 && j < len(row) { + switch { + case j >= 0 && j < len(row): value, err = table.CastValue(kvcodec.se, row[j], col.ToInfo(), false, false) if err == nil { value, err = col.HandleBadNull(value, kvcodec.se.vars.StmtCtx) } - } else if isAutoIncCol { + case isAutoIncCol: // we still need a conversion, e.g. to catch overflow with a TINYINT column. value, err = table.CastValue(kvcodec.se, types.NewIntDatum(rowID), col.ToInfo(), false, false) - } else if isAutoRandom && isPk { + case isAutoRandom && isPk: var val types.Datum if mysql.HasUnsignedFlag(col.Flag) { val = types.NewUintDatum(uint64(kvcodec.autoRandomHeaderBits | rowID)) @@ -314,11 +329,11 @@ func (kvcodec *tableKVEncoder) Encode( val = types.NewIntDatum(kvcodec.autoRandomHeaderBits | rowID) } value, err = table.CastValue(kvcodec.se, val, col.ToInfo(), false, false) - } else if col.IsGenerated() { + case col.IsGenerated(): // inject some dummy value for gen col so that MutRowFromDatums below sees a real value instead of nil. // if MutRowFromDatums sees a nil it won't initialize the underlying storage and cause SetDatum to panic. value = types.GetMinValue(&col.FieldType) - } else { + default: value, err = table.GetColDefaultValue(kvcodec.se, col.ToInfo()) } if err != nil { @@ -329,10 +344,14 @@ func (kvcodec *tableKVEncoder) Encode( if isAutoRandom && isPk { incrementalBits := autoRandomIncrementBits(col, int(kvcodec.tbl.Meta().AutoRandomBits)) - kvcodec.tbl.RebaseAutoID(kvcodec.se, value.GetInt64()&((1< 0 { @@ -404,8 +425,8 @@ func (kvs kvPairs) ClassifyAndAppend( *indices = indexKVs } -func (totalKVs kvPairs) SplitIntoChunks(splitSize int) []Rows { - if len(totalKVs) == 0 { +func (kvs kvPairs) SplitIntoChunks(splitSize int) []Rows { + if len(kvs) == 0 { return nil } @@ -413,19 +434,19 @@ func (totalKVs kvPairs) SplitIntoChunks(splitSize int) []Rows { i := 0 cumSize := 0 - for j, pair := range totalKVs { + for j, pair := range kvs { size := len(pair.Key) + len(pair.Val) if i < j && cumSize+size > splitSize { - res = append(res, kvPairs(totalKVs[i:j])) + res = append(res, kvs[i:j]) i = j cumSize = 0 } cumSize += size } - return append(res, kvPairs(totalKVs[i:])) + return append(res, kvs[i:]) } func (kvs kvPairs) Clear() Rows { - return kvPairs(kvs[:0]) + return kvs[:0] } diff --git a/pkg/lightning/backend/sql2kv_test.go b/pkg/lightning/backend/sql2kv_test.go index d2b84b07f..d78204618 100644 --- a/pkg/lightning/backend/sql2kv_test.go +++ b/pkg/lightning/backend/sql2kv_test.go @@ -94,7 +94,7 @@ func (s *kvSuite) TestEncode(c *C) { types.NewIntDatum(1), types.NewStringDatum("invalid-pk"), } - pairs, err = strictMode.Encode(logger, rowsWithPk, 2, []int{0, 1}) + _, err = strictMode.Encode(logger, rowsWithPk, 2, []int{0, 1}) c.Assert(err, ErrorMatches, "failed to cast value as bigint\\(20\\) for column `_tidb_rowid`.*Truncated.*") rowsWithPk2 := []types.Datum{ @@ -117,7 +117,7 @@ func (s *kvSuite) TestEncode(c *C) { Timestamp: 1234567891, }) c.Assert(err, IsNil) - pairs, err = mockMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}) + _, err = mockMode.Encode(logger, rowsWithPk2, 2, []int{0, 1}) c.Assert(err, ErrorMatches, "mock error") // Non-strict mode @@ -214,9 +214,9 @@ func (s *kvSuite) TestEncodeTimestamp(c *C) { })) } -func mockTableInfo(c *C, createSql string) *model.TableInfo { +func mockTableInfo(c *C, createSQL string) *model.TableInfo { parser := parser.New() - node, err := parser.ParseOneStmt(createSql, "", "") + node, err := parser.ParseOneStmt(createSQL, "", "") c.Assert(err, IsNil) sctx := mock.NewContext() info, err := ddl.MockTableInfo(sctx, node.(*ast.CreateTableStmt), 1) @@ -398,6 +398,7 @@ func (s *benchSQL2KVSuite) SetUpTest(c *C) { tbl, err := tables.TableFromMeta(NewPanickingAllocators(0), tableInfo) c.Assert(err, IsNil) s.encoder, err = NewTableKVEncoder(tbl, &SessionOptions{SysVars: map[string]string{"tidb_row_format_version": "2"}}) + c.Assert(err, IsNil) s.logger = log.Logger{Logger: zap.NewNop()} // Prepare the row to insert. diff --git a/pkg/lightning/backend/tidb.go b/pkg/lightning/backend/tidb.go index 4d968b34b..595db6d0d 100644 --- a/pkg/lightning/backend/tidb.go +++ b/pkg/lightning/backend/tidb.go @@ -50,8 +50,8 @@ type tidbRow string type tidbRows []tidbRow // MarshalLogArray implements the zapcore.ArrayMarshaler interface -func (row tidbRows) MarshalLogArray(encoder zapcore.ArrayEncoder) error { - for _, r := range row { +func (rows tidbRows) MarshalLogArray(encoder zapcore.ArrayEncoder) error { + for _, r := range rows { encoder.AppendString(string(r)) } return nil @@ -88,7 +88,9 @@ func NewTiDBBackend(db *sql.DB, onDuplicate string) Backend { func (row tidbRow) ClassifyAndAppend(data *Rows, checksum *verification.KVChecksum, _ *Rows, _ *verification.KVChecksum) { rows := (*data).(tidbRows) - *data = tidbRows(append(rows, row)) + // Cannot do `rows := data.(*tidbRows); *rows = append(*rows, row)`. + //nolint:gocritic + *data = append(rows, row) cs := verification.MakeKVChecksum(uint64(len(row)), 1, 0) checksum.Add(&cs) } @@ -158,7 +160,7 @@ func (enc *tidbEncoder) appendSQLBytes(sb *strings.Builder, value []byte) { // appendSQL appends the SQL representation of the Datum into the string builder. // Note that we cannot use Datum.ToString since it doesn't perform SQL escaping. -func (enc *tidbEncoder) appendSQL(sb *strings.Builder, datum *types.Datum, col *table.Column) error { +func (enc *tidbEncoder) appendSQL(sb *strings.Builder, datum *types.Datum, _ *table.Column) error { switch datum.Kind() { case types.KindNull: sb.WriteString("NULL") @@ -188,13 +190,13 @@ func (enc *tidbEncoder) appendSQL(sb *strings.Builder, datum *types.Datum, col * sb.Write(value) case types.KindString: // See: https://github.com/pingcap/tidb-lightning/issues/550 - //if enc.mode.HasStrictMode() { + // if enc.mode.HasStrictMode() { // d, err := table.CastValue(enc.se, *datum, col.ToInfo(), false, false) // if err != nil { // return errors.Trace(err) // } // datum = &d - //} + // } enc.appendSQLBytes(sb, datum.GetBytes()) case types.KindBytes: @@ -211,7 +213,9 @@ func (enc *tidbEncoder) appendSQL(sb *strings.Builder, datum *types.Datum, col * value := datum.GetBinaryLiteral() sb.Grow(3 + 2*len(value)) sb.WriteString("x'") - hex.NewEncoder(sb).Write(value) + if _, err := hex.NewEncoder(sb).Write(value); err != nil { + return errors.Trace(err) + } sb.WriteByte('\'') case types.KindMysqlBit: @@ -278,7 +282,8 @@ func (enc *tidbEncoder) Encode(logger log.Logger, row []types.Datum, _ int64, co if i != 0 { encoded.WriteByte(',') } - if err := enc.appendSQL(&encoded, &field, getColumnByIndex(cols, enc.columnIdx[i])); err != nil { + datum := field + if err := enc.appendSQL(&encoded, &datum, getColumnByIndex(cols, enc.columnIdx[i])); err != nil { logger.Error("tidb encode failed", zap.Array("original", rowArrayMarshaler(row)), zap.Int("originalCol", i), @@ -416,6 +421,7 @@ func (be *tidbBackend) WriteRowsToDB(ctx context.Context, tableName string, colu return errors.Trace(err) } +//nolint:nakedret // TODO: refactor func (be *tidbBackend) FetchRemoteTableModels(ctx context.Context, schemaName string) (tables []*model.TableInfo, err error) { s := common.SQLWithRetry{ DB: be.db, @@ -506,11 +512,11 @@ func (be *tidbBackend) FetchRemoteTableModels(ctx context.Context, schemaName st return err } - //+--------------+------------+-------------+--------------------+----------------+ - //| DB_NAME | TABLE_NAME | COLUMN_NAME | NEXT_GLOBAL_ROW_ID | ID_TYPE | - //+--------------+------------+-------------+--------------------+----------------+ - //| testsysbench | t | _tidb_rowid | 1 | AUTO_INCREMENT | - //+--------------+------------+-------------+--------------------+----------------+ + // +--------------+------------+-------------+--------------------+----------------+ + // | DB_NAME | TABLE_NAME | COLUMN_NAME | NEXT_GLOBAL_ROW_ID | ID_TYPE | + // +--------------+------------+-------------+--------------------+----------------+ + // | testsysbench | t | _tidb_rowid | 1 | AUTO_INCREMENT | + // +--------------+------------+-------------+--------------------+----------------+ // if columns length is 4, it doesn't contains the last column `ID_TYPE`, and it will always be 'AUTO_INCREMENT' // for v4.0.0~v4.0.2 show table t next_row_id only returns 4 columns. @@ -538,9 +544,13 @@ func (be *tidbBackend) FetchRemoteTableModels(ctx context.Context, schemaName st } } } - rows.Close() + // Defer in for-loop would be costly, anyway, we don't need those rows after this turn of iteration. + //nolint:sqlclosecheck + if err := rows.Close(); err != nil { + return errors.Trace(err) + } if rows.Err() != nil { - return rows.Err() + return errors.Trace(rows.Err()) } } return nil @@ -564,19 +574,28 @@ func (be *tidbBackend) ResetEngine(context.Context, uuid.UUID) error { return errors.New("cannot reset an engine in TiDB backend") } +<<<<<<< HEAD:pkg/lightning/backend/tidb.go func (be *tidbBackend) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (EngineWriter, error) { return &TiDBWriter{be: be, engineUUID: engineUUID}, nil +======= +func (be *tidbBackend) LocalWriter(ctx context.Context, engineUUID uuid.UUID) (backend.EngineWriter, error) { + return &Writer{be: be, engineUUID: engineUUID}, nil +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/tidb/tidb.go } -type TiDBWriter struct { +type Writer struct { be *tidbBackend engineUUID uuid.UUID } -func (w *TiDBWriter) Close() error { +func (w *Writer) Close() error { return nil } +<<<<<<< HEAD:pkg/lightning/backend/tidb.go func (w *TiDBWriter) AppendRows(ctx context.Context, tableName string, columnNames []string, arg1 uint64, rows Rows) error { +======= +func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, arg1 uint64, rows kv.Rows) error { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766):pkg/lightning/backend/tidb/tidb.go return w.be.WriteRows(ctx, w.engineUUID, tableName, columnNames, arg1, rows) } diff --git a/pkg/lightning/backend/tidb_test.go b/pkg/lightning/backend/tidb_test.go index 4603419be..fb5e36945 100644 --- a/pkg/lightning/backend/tidb_test.go +++ b/pkg/lightning/backend/tidb_test.go @@ -156,11 +156,13 @@ func (s *mysqlSuite) TestWriteRowsIgnoreOnDup(c *C) { // test encode rows with _tidb_rowid encoder, err = ignoreBackend.NewEncoder(s.tbl, &kv.SessionOptions{}) c.Assert(err, IsNil) - row, err = encoder.Encode(logger, []types.Datum{ + rowWithID, err := encoder.Encode(logger, []types.Datum{ types.NewIntDatum(1), types.NewIntDatum(1), // _tidb_rowid field }, 1, []int{0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1}) c.Assert(err, IsNil) + // tidbRow is string. + c.Assert(fmt.Sprint(rowWithID), Equals, "(1,1)") } func (s *mysqlSuite) TestWriteRowsErrorOnDup(c *C) { @@ -198,6 +200,7 @@ func (s *mysqlSuite) TestWriteRowsErrorOnDup(c *C) { } // TODO: temporarily disable this test before we fix strict mode +//nolint:unused func (s *mysqlSuite) testStrictMode(c *C) { ft := *types.NewFieldType(mysql.TypeVarchar) ft.Charset = charset.CharsetUTF8MB4 diff --git a/pkg/lightning/checkpoints/checkpoints.go b/pkg/lightning/checkpoints/checkpoints.go index 75d694cbe..bcc64791c 100644 --- a/pkg/lightning/checkpoints/checkpoints.go +++ b/pkg/lightning/checkpoints/checkpoints.go @@ -65,6 +65,11 @@ const ( CheckpointTableNameTable = "table_v6" CheckpointTableNameEngine = "engine_v5" CheckpointTableNameChunk = "chunk_v5" + + // Some frequently used table name or constants. + allTables = "all" + stringLitAll = "'all'" + columnTableName = "table_name" ) const ( @@ -360,14 +365,15 @@ func (cp *TableCheckpoint) Apply(cpd *TableCheckpointDiff) { engine.Status = engineDiff.status } for key, diff := range engineDiff.chunks { + checkpointKey := key index := sort.Search(len(engine.Chunks), func(i int) bool { - return !engine.Chunks[i].Key.less(&key) + return !engine.Chunks[i].Key.less(&checkpointKey) }) if index >= len(engine.Chunks) { continue } chunk := engine.Chunks[index] - if chunk.Key != key { + if chunk.Key != checkpointKey { continue } chunk.Chunk.Offset = diff.pos @@ -447,7 +453,7 @@ type DestroyedTableCheckpoint struct { } type TaskCheckpoint struct { - TaskId int64 + TaskID int64 SourceDir string Backend string ImporterAddr string @@ -458,7 +464,7 @@ type TaskCheckpoint struct { LightningVer string } -type CheckpointsDB interface { +type DB interface { Initialize(ctx context.Context, cfg *config.Config, dbInfo map[string]*TidbDBInfo) error TaskCheckpoint(ctx context.Context) (*TaskCheckpoint, error) Get(ctx context.Context, tableName string) (*TableCheckpoint, error) @@ -483,7 +489,7 @@ type CheckpointsDB interface { DumpChunks(ctx context.Context, csv io.Writer) error } -func OpenCheckpointsDB(ctx context.Context, cfg *config.Config) (CheckpointsDB, error) { +func OpenCheckpointsDB(ctx context.Context, cfg *config.Config) (DB, error) { if !cfg.Checkpoint.Enable { return NewNullCheckpointsDB(), nil } @@ -526,7 +532,11 @@ func IsCheckpointsDBExists(ctx context.Context, cfg *config.Config) (bool, error return false, errors.Trace(err) } defer rows.Close() - return rows.Next(), nil + result := rows.Next() + if err := rows.Err(); err != nil { + return false, errors.Trace(err) + } + return result, nil case config.CheckpointDriverFile: _, err := os.Stat(cfg.Checkpoint.DSN) @@ -676,7 +686,7 @@ func (cpdb *MySQLCheckpointsDB) TaskCheckpoint(ctx context.Context) (*TaskCheckp taskQuery := fmt.Sprintf(ReadTaskTemplate, cpdb.schema, CheckpointTableNameTask) taskCp := &TaskCheckpoint{} - err := s.QueryRow(ctx, "fetch task checkpoint", taskQuery, &taskCp.TaskId, &taskCp.SourceDir, &taskCp.Backend, + err := s.QueryRow(ctx, "fetch task checkpoint", taskQuery, &taskCp.TaskID, &taskCp.SourceDir, &taskCp.Backend, &taskCp.ImporterAddr, &taskCp.TiDBHost, &taskCp.TiDBPort, &taskCp.PdAddr, &taskCp.SortedKVDir, &taskCp.LightningVer) if err != nil { // if task checkpoint is empty, return nil @@ -1007,7 +1017,7 @@ func (cpdb *FileCheckpointsDB) TaskCheckpoint(_ context.Context) (*TaskCheckpoin } return &TaskCheckpoint{ - TaskId: cp.TaskId, + TaskID: cp.TaskId, SourceDir: cp.SourceDir, Backend: cp.Backend, ImporterAddr: cp.ImporterAddr, @@ -1119,7 +1129,6 @@ func (cpdb *FileCheckpointsDB) InsertEngineCheckpoints(_ context.Context, tableN if len(value.ColumnPermutation) > 0 { chunk.ColumnPermutation = intSlice2Int32Slice(value.ColumnPermutation) } - } tableModel.Engines[engineID] = engineModel } @@ -1164,14 +1173,14 @@ func (cpdb *FileCheckpointsDB) Update(checkpointDiffs map[string]*TableCheckpoin // Management functions ---------------------------------------------------------------------------- -var cannotManageNullDB = errors.New("cannot perform this function while checkpoints is disabled") +var errCannotManageNullDB = errors.New("cannot perform this function while checkpoints is disabled") func (*NullCheckpointsDB) RemoveCheckpoint(context.Context, string) error { - return errors.Trace(cannotManageNullDB) + return errors.Trace(errCannotManageNullDB) } func (*NullCheckpointsDB) MoveCheckpoints(context.Context, int64) error { - return errors.Trace(cannotManageNullDB) + return errors.Trace(errCannotManageNullDB) } func (*NullCheckpointsDB) GetLocalStoringTables(context.Context) (map[string][]int32, error) { @@ -1179,23 +1188,23 @@ func (*NullCheckpointsDB) GetLocalStoringTables(context.Context) (map[string][]i } func (*NullCheckpointsDB) IgnoreErrorCheckpoint(context.Context, string) error { - return errors.Trace(cannotManageNullDB) + return errors.Trace(errCannotManageNullDB) } func (*NullCheckpointsDB) DestroyErrorCheckpoint(context.Context, string) ([]DestroyedTableCheckpoint, error) { - return nil, errors.Trace(cannotManageNullDB) + return nil, errors.Trace(errCannotManageNullDB) } func (*NullCheckpointsDB) DumpTables(context.Context, io.Writer) error { - return errors.Trace(cannotManageNullDB) + return errors.Trace(errCannotManageNullDB) } func (*NullCheckpointsDB) DumpEngines(context.Context, io.Writer) error { - return errors.Trace(cannotManageNullDB) + return errors.Trace(errCannotManageNullDB) } func (*NullCheckpointsDB) DumpChunks(context.Context, io.Writer) error { - return errors.Trace(cannotManageNullDB) + return errors.Trace(errCannotManageNullDB) } func (cpdb *MySQLCheckpointsDB) RemoveCheckpoint(ctx context.Context, tableName string) error { @@ -1204,7 +1213,7 @@ func (cpdb *MySQLCheckpointsDB) RemoveCheckpoint(ctx context.Context, tableName Logger: log.With(zap.String("table", tableName)), } - if tableName == "all" { + if tableName == allTables { return s.Exec(ctx, "remove all checkpoints", "DROP SCHEMA "+cpdb.schema) } @@ -1303,12 +1312,12 @@ func (cpdb *MySQLCheckpointsDB) GetLocalStoringTables(ctx context.Context) (map[ func (cpdb *MySQLCheckpointsDB) IgnoreErrorCheckpoint(ctx context.Context, tableName string) error { var colName string - if tableName == "all" { + if tableName == allTables { // This will expand to `WHERE 'all' = 'all'` and effectively allowing // all tables to be included. - colName = "'all'" + colName = stringLitAll } else { - colName = "table_name" + colName = columnTableName } engineQuery := fmt.Sprintf(` @@ -1337,13 +1346,13 @@ func (cpdb *MySQLCheckpointsDB) IgnoreErrorCheckpoint(ctx context.Context, table func (cpdb *MySQLCheckpointsDB) DestroyErrorCheckpoint(ctx context.Context, tableName string) ([]DestroyedTableCheckpoint, error) { var colName, aliasedColName string - if tableName == "all" { + if tableName == allTables { // These will expand to `WHERE 'all' = 'all'` and effectively allowing // all tables to be included. - colName = "'all'" - aliasedColName = "'all'" + colName = stringLitAll + aliasedColName = stringLitAll } else { - colName = "table_name" + colName = columnTableName aliasedColName = "t.table_name" } @@ -1411,6 +1420,7 @@ func (cpdb *MySQLCheckpointsDB) DestroyErrorCheckpoint(ctx context.Context, tabl return targetTables, nil } +//nolint:rowserrcheck // sqltocsv.Write will check this. func (cpdb *MySQLCheckpointsDB) DumpTables(ctx context.Context, writer io.Writer) error { rows, err := cpdb.db.QueryContext(ctx, fmt.Sprintf(` SELECT @@ -1431,6 +1441,7 @@ func (cpdb *MySQLCheckpointsDB) DumpTables(ctx context.Context, writer io.Writer return errors.Trace(sqltocsv.Write(writer, rows)) } +//nolint:rowserrcheck // sqltocsv.Write will check this. func (cpdb *MySQLCheckpointsDB) DumpEngines(ctx context.Context, writer io.Writer) error { rows, err := cpdb.db.QueryContext(ctx, fmt.Sprintf(` SELECT @@ -1449,6 +1460,7 @@ func (cpdb *MySQLCheckpointsDB) DumpEngines(ctx context.Context, writer io.Write return errors.Trace(sqltocsv.Write(writer, rows)) } +//nolint:rowserrcheck // sqltocsv.Write will check this. func (cpdb *MySQLCheckpointsDB) DumpChunks(ctx context.Context, writer io.Writer) error { rows, err := cpdb.db.QueryContext(ctx, fmt.Sprintf(` SELECT @@ -1483,7 +1495,7 @@ func (cpdb *FileCheckpointsDB) RemoveCheckpoint(_ context.Context, tableName str cpdb.lock.Lock() defer cpdb.lock.Unlock() - if tableName == "all" { + if tableName == allTables { cpdb.checkpoints.Reset() return errors.Trace(os.Remove(cpdb.path)) } @@ -1523,7 +1535,6 @@ func (cpdb *FileCheckpointsDB) GetLocalStoringTables(_ context.Context) (map[str break } } - } } @@ -1535,7 +1546,7 @@ func (cpdb *FileCheckpointsDB) IgnoreErrorCheckpoint(_ context.Context, targetTa defer cpdb.lock.Unlock() for tableName, tableModel := range cpdb.checkpoints.Checkpoints { - if !(targetTableName == "all" || targetTableName == tableName) { + if !(targetTableName == allTables || targetTableName == tableName) { continue } if tableModel.Status <= uint32(CheckpointStatusMaxInvalid) { @@ -1558,7 +1569,7 @@ func (cpdb *FileCheckpointsDB) DestroyErrorCheckpoint(_ context.Context, targetT for tableName, tableModel := range cpdb.checkpoints.Checkpoints { // Obtain the list of tables - if !(targetTableName == "all" || targetTableName == tableName) { + if !(targetTableName == allTables || targetTableName == tableName) { continue } if tableModel.Status <= uint32(CheckpointStatusMaxInvalid) { diff --git a/pkg/lightning/checkpoints/checkpoints_file_test.go b/pkg/lightning/checkpoints/checkpoints_file_test.go index e49a9d738..25ced0040 100644 --- a/pkg/lightning/checkpoints/checkpoints_file_test.go +++ b/pkg/lightning/checkpoints/checkpoints_file_test.go @@ -21,9 +21,7 @@ func Test(t *testing.T) { var _ = Suite(&cpFileSuite{}) type cpFileSuite struct { - path string cpdb *checkpoints.FileCheckpointsDB - cfg *config.Config } func newTestConfig() *config.Config { diff --git a/pkg/lightning/checkpoints/glue_checkpoint.go b/pkg/lightning/checkpoints/glue_checkpoint.go index 82c3bc5b0..5eb869c20 100644 --- a/pkg/lightning/checkpoints/glue_checkpoint.go +++ b/pkg/lightning/checkpoints/glue_checkpoint.go @@ -55,7 +55,15 @@ type GlueCheckpointsDB struct { schema string } -var _ CheckpointsDB = (*GlueCheckpointsDB)(nil) +var _ DB = (*GlueCheckpointsDB)(nil) + +// dropPreparedStmt drops the statement and when meet an error, +// print an error message. +func dropPreparedStmt(session Session, stmtID uint32) { + if err := session.DropPreparedStmt(stmtID); err != nil { + log.L().Error("failed to drop prepared statement", log.ShortError(err)) + } +} func NewGlueCheckpointsDB(ctx context.Context, se Session, f func() (Session, error), schemaName string) (*GlueCheckpointsDB, error) { var escapedSchemaName strings.Builder @@ -127,7 +135,7 @@ func (g GlueCheckpointsDB) Initialize(ctx context.Context, cfg *config.Config, d if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(stmtID) + defer dropPreparedStmt(s, stmtID) _, err = s.ExecutePreparedStmt(c, stmtID, []types.Datum{ types.NewIntDatum(cfg.TaskID), types.NewStringDatum(cfg.Mydumper.SourceDir), @@ -147,7 +155,7 @@ func (g GlueCheckpointsDB) Initialize(ctx context.Context, cfg *config.Config, d if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(stmtID2) + defer dropPreparedStmt(s, stmtID2) for _, db := range dbInfo { for _, table := range db.Tables { @@ -196,7 +204,7 @@ func (g GlueCheckpointsDB) TaskCheckpoint(ctx context.Context) (*TaskCheckpoint, row := req.GetRow(0) taskCp = &TaskCheckpoint{} - taskCp.TaskId = row.GetInt64(0) + taskCp.TaskID = row.GetInt64(0) taskCp.SourceDir = row.GetString(1) taskCp.Backend = row.GetString(2) taskCp.ImporterAddr = row.GetString(3) @@ -355,13 +363,13 @@ func (g GlueCheckpointsDB) InsertEngineCheckpoints(ctx context.Context, tableNam if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(engineStmt) + defer dropPreparedStmt(s, engineStmt) chunkStmt, _, _, err := s.PrepareStmt(fmt.Sprintf(ReplaceChunkTemplate, g.schema, CheckpointTableNameChunk)) if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(chunkStmt) + defer dropPreparedStmt(s, chunkStmt) for engineID, engine := range checkpointMap { _, err := s.ExecutePreparedStmt(c, engineStmt, []types.Datum{ @@ -421,22 +429,22 @@ func (g GlueCheckpointsDB) Update(checkpointDiffs map[string]*TableCheckpointDif if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(chunkStmt) + defer dropPreparedStmt(s, chunkStmt) rebaseStmt, _, _, err := s.PrepareStmt(rebaseQuery) if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(rebaseStmt) + defer dropPreparedStmt(s, rebaseStmt) tableStatusStmt, _, _, err := s.PrepareStmt(tableStatusQuery) if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(tableStatusStmt) + defer dropPreparedStmt(s, tableStatusStmt) engineStatusStmt, _, _, err := s.PrepareStmt(engineStatusQuery) if err != nil { return errors.Trace(err) } - defer s.DropPreparedStmt(engineStatusStmt) + defer dropPreparedStmt(s, engineStatusStmt) for tableName, cpd := range checkpointDiffs { if cpd.hasStatus { @@ -506,7 +514,7 @@ func (g GlueCheckpointsDB) RemoveCheckpoint(ctx context.Context, tableName strin } defer se.Close() - if tableName == "all" { + if tableName == allTables { return common.Retry("remove all checkpoints", logger, func() error { _, err := se.Execute(ctx, "DROP SCHEMA "+g.schema) return err @@ -625,12 +633,12 @@ func (g GlueCheckpointsDB) IgnoreErrorCheckpoint(ctx context.Context, tableName defer se.Close() var colName string - if tableName == "all" { + if tableName == allTables { // This will expand to `WHERE 'all' = 'all'` and effectively allowing // all tables to be included. - colName = "'all'" + colName = stringLitAll } else { - colName = "table_name" + colName = columnTableName } tableName = common.InterpolateMySQLString(tableName) @@ -662,13 +670,13 @@ func (g GlueCheckpointsDB) DestroyErrorCheckpoint(ctx context.Context, tableName var colName, aliasedColName string - if tableName == "all" { + if tableName == allTables { // These will expand to `WHERE 'all' = 'all'` and effectively allowing // all tables to be included. - colName = "'all'" - aliasedColName = "'all'" + colName = stringLitAll + aliasedColName = stringLitAll } else { - colName = "table_name" + colName = columnTableName aliasedColName = "t.table_name" } diff --git a/pkg/lightning/common/pause_test.go b/pkg/lightning/common/pause_test.go index cb7d2264b..298e023cd 100644 --- a/pkg/lightning/common/pause_test.go +++ b/pkg/lightning/common/pause_test.go @@ -147,7 +147,7 @@ func (s *pauseSuite) BenchmarkWaitNoOp(c *C) { p := common.NewPauser() ctx := context.Background() for i := 0; i < c.N; i++ { - p.Wait(ctx) + _ = p.Wait(ctx) } } @@ -157,7 +157,7 @@ func (s *pauseSuite) BenchmarkWaitCtxCanceled(c *C) { ctx, cancel := context.WithCancel(context.Background()) cancel() for i := 0; i < c.N; i++ { - p.Wait(ctx) + _ = p.Wait(ctx) } } @@ -185,6 +185,6 @@ func (s *pauseSuite) BenchmarkWaitContended(c *C) { ctx := context.Background() for i := 0; i < c.N; i++ { - p.Wait(ctx) + _ = p.Wait(ctx) } } diff --git a/pkg/lightning/common/security_test.go b/pkg/lightning/common/security_test.go index e7db1f86d..bf9879abd 100644 --- a/pkg/lightning/common/security_test.go +++ b/pkg/lightning/common/security_test.go @@ -32,9 +32,9 @@ type securitySuite struct{} var _ = Suite(&securitySuite{}) func respondPathHandler(w http.ResponseWriter, req *http.Request) { - io.WriteString(w, `{"path":"`) - io.WriteString(w, req.URL.Path) - io.WriteString(w, `"}`) + _, _ = io.WriteString(w, `{"path":"`) + _, _ = io.WriteString(w, req.URL.Path) + _, _ = io.WriteString(w, `"}`) } func (s *securitySuite) TestGetJSONInsecure(c *C) { diff --git a/pkg/lightning/common/util.go b/pkg/lightning/common/util.go index b9d4013f7..f3b6f8059 100644 --- a/pkg/lightning/common/util.go +++ b/pkg/lightning/common/util.go @@ -106,7 +106,7 @@ type SQLWithRetry struct { HideQueryLog bool } -func (t SQLWithRetry) perform(ctx context.Context, parentLogger log.Logger, purpose string, action func() error) error { +func (t SQLWithRetry) perform(_ context.Context, parentLogger log.Logger, purpose string, action func() error) error { return Retry(purpose, parentLogger, action) } diff --git a/pkg/lightning/common/util_test.go b/pkg/lightning/common/util_test.go index 3fab74640..17998de20 100644 --- a/pkg/lightning/common/util_test.go +++ b/pkg/lightning/common/util_test.go @@ -57,7 +57,7 @@ func (s *utilSuite) TestGetJSON(c *C) { ctx := context.Background() // Mock success response - handle := func(res http.ResponseWriter, req *http.Request) { + handle := func(res http.ResponseWriter, _ *http.Request) { res.WriteHeader(http.StatusOK) err := json.NewEncoder(res).Encode(request) c.Assert(err, IsNil) @@ -77,7 +77,7 @@ func (s *utilSuite) TestGetJSON(c *C) { c.Assert(request, DeepEquals, response) // Mock `StatusNoContent` response - handle = func(res http.ResponseWriter, req *http.Request) { + handle = func(res http.ResponseWriter, _ *http.Request) { res.WriteHeader(http.StatusNoContent) } err = common.GetJSON(ctx, client, testServer.URL, &response) diff --git a/pkg/lightning/config/config.go b/pkg/lightning/config/config.go index e3b93aa23..a539d9da9 100644 --- a/pkg/lightning/config/config.go +++ b/pkg/lightning/config/config.go @@ -71,9 +71,6 @@ const ( defaultIndexSerialScanConcurrency = 20 defaultChecksumTableConcurrency = 2 - defaultEngineMemCacheSize = 512 * units.MiB - defaultLocalWriterMemCacheSize = 128 * units.MiB - // autoDiskQuotaLocalReservedSpeed is the estimated size increase per // millisecond per write thread the local backend may gain on all engines. // This is used to compute the maximum size overshoot between two disk quota @@ -82,10 +79,11 @@ const ( // With cron.check-disk-quota = 1m, region-concurrency = 40, this should // contribute 2.3 GiB to the reserved size. autoDiskQuotaLocalReservedSpeed uint64 = 1 * units.KiB + defaultEngineMemCacheSize = 512 * units.MiB + defaultLocalWriterMemCacheSize = 128 * units.MiB ) var ( - defaultConfigPaths = []string{"tidb-lightning.toml", "conf/tidb-lightning.toml"} supportedStorageTypes = []string{"file", "local", "s3", "noop"} DefaultFilter = []string{ @@ -136,17 +134,17 @@ type Config struct { BWList filter.MySQLReplicationRules `toml:"black-white-list" json:"black-white-list"` } -func (c *Config) String() string { - bytes, err := json.Marshal(c) +func (cfg *Config) String() string { + bytes, err := json.Marshal(cfg) if err != nil { log.L().Error("marshal config to json error", log.ShortError(err)) } return string(bytes) } -func (c *Config) ToTLS() (*common.TLS, error) { - hostPort := net.JoinHostPort(c.TiDB.Host, strconv.Itoa(c.TiDB.StatusPort)) - return common.NewTLS(c.Security.CAPath, c.Security.CertPath, c.Security.KeyPath, hostPort) +func (cfg *Config) ToTLS() (*common.TLS, error) { + hostPort := net.JoinHostPort(cfg.TiDB.Host, strconv.Itoa(cfg.TiDB.StatusPort)) + return common.NewTLS(cfg.Security.CAPath, cfg.Security.CertPath, cfg.Security.KeyPath, hostPort) } type Lightning struct { @@ -188,6 +186,7 @@ func (t PostOpLevel) MarshalText() ([]byte, error) { // parser command line parameter func (t *PostOpLevel) FromStringValue(s string) error { switch strings.ToLower(s) { + //nolint:goconst // This 'false' and other 'false's aren't the same. case "off", "false": *t = OpLevelOff case "required", "true": @@ -223,20 +222,20 @@ func (t PostOpLevel) String() string { // PostRestore has some options which will be executed after kv restored. type PostRestore struct { - Level1Compact bool `toml:"level-1-compact" json:"level-1-compact"` - Compact bool `toml:"compact" json:"compact"` Checksum PostOpLevel `toml:"checksum" json:"checksum"` Analyze PostOpLevel `toml:"analyze" json:"analyze"` + Level1Compact bool `toml:"level-1-compact" json:"level-1-compact"` PostProcessAtLast bool `toml:"post-process-at-last" json:"post-process-at-last"` + Compact bool `toml:"compact" json:"compact"` } type CSVConfig struct { Separator string `toml:"separator" json:"separator"` Delimiter string `toml:"delimiter" json:"delimiter"` + Null string `toml:"null" json:"null"` Header bool `toml:"header" json:"header"` TrimLastSep bool `toml:"trim-last-separator" json:"trim-last-separator"` NotNull bool `toml:"not-null" json:"not-null"` - Null string `toml:"null" json:"null"` BackslashEscape bool `toml:"backslash-escape" json:"backslash-escape"` } @@ -245,14 +244,14 @@ type MydumperRuntime struct { BatchSize ByteSize `toml:"batch-size" json:"batch-size"` BatchImportRatio float64 `toml:"batch-import-ratio" json:"batch-import-ratio"` SourceDir string `toml:"data-source-dir" json:"data-source-dir"` - NoSchema bool `toml:"no-schema" json:"no-schema"` CharacterSet string `toml:"character-set" json:"character-set"` CSV CSVConfig `toml:"csv" json:"csv"` - CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"` - StrictFormat bool `toml:"strict-format" json:"strict-format"` MaxRegionSize ByteSize `toml:"max-region-size" json:"max-region-size"` Filter []string `toml:"filter" json:"filter"` FileRouters []*FileRouteRule `toml:"files" json:"files"` + NoSchema bool `toml:"no-schema" json:"no-schema"` + CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"` + StrictFormat bool `toml:"strict-format" json:"strict-format"` DefaultFileRules bool `toml:"default-file-rules" json:"default-file-rules"` } @@ -282,10 +281,10 @@ type TikvImporter struct { } type Checkpoint struct { - Enable bool `toml:"enable" json:"enable"` Schema string `toml:"schema" json:"schema"` DSN string `toml:"dsn" json:"-"` // DSN may contain password, don't expose this to JSON. Driver string `toml:"driver" json:"driver"` + Enable bool `toml:"enable" json:"enable"` KeepAfterSuccess bool `toml:"keep-after-success" json:"keep-after-success"` } @@ -312,7 +311,7 @@ func (sec *Security) RegisterMySQL() error { tlsConfig, err := common.ToTLSConfig(sec.CAPath, sec.CertPath, sec.KeyPath) switch { case err != nil: - return err + return errors.Trace(err) case tlsConfig != nil: // error happens only when the key coincides with the built-in names. _ = gomysql.RegisterTLSConfig("cluster", tlsConfig) @@ -331,7 +330,7 @@ type Duration struct { func (d *Duration) UnmarshalText(text []byte) error { var err error d.Duration, err = time.ParseDuration(string(text)) - return err + return errors.Trace(err) } func (d Duration) MarshalText() ([]byte, error) { @@ -533,38 +532,10 @@ func (cfg *Config) Adjust(ctx context.Context) error { mustHaveInternalConnections := true switch cfg.TikvImporter.Backend { case BackendTiDB: - if cfg.App.IndexConcurrency == 0 { - cfg.App.IndexConcurrency = cfg.App.RegionConcurrency - } - if cfg.App.TableConcurrency == 0 { - cfg.App.TableConcurrency = cfg.App.RegionConcurrency - } + cfg.DefaultVarsForTiDBBackend() mustHaveInternalConnections = false case BackendImporter, BackendLocal: - if cfg.App.IndexConcurrency == 0 { - cfg.App.IndexConcurrency = 2 - } - if cfg.App.TableConcurrency == 0 { - cfg.App.TableConcurrency = 6 - } - if cfg.TikvImporter.RangeConcurrency == 0 { - cfg.TikvImporter.RangeConcurrency = 16 - } - if cfg.TikvImporter.RegionSplitSize == 0 { - cfg.TikvImporter.RegionSplitSize = SplitRegionSize - } - if cfg.TiDB.DistSQLScanConcurrency == 0 { - cfg.TiDB.DistSQLScanConcurrency = defaultDistSQLScanConcurrency - } - if cfg.TiDB.BuildStatsConcurrency == 0 { - cfg.TiDB.BuildStatsConcurrency = defaultBuildStatsConcurrency - } - if cfg.TiDB.IndexSerialScanConcurrency == 0 { - cfg.TiDB.IndexSerialScanConcurrency = defaultIndexSerialScanConcurrency - } - if cfg.TiDB.ChecksumTableConcurrency == 0 { - cfg.TiDB.ChecksumTableConcurrency = defaultChecksumTableConcurrency - } + cfg.DefaultVarsForImporterAndLocalBackend() default: return errors.Errorf("invalid config: unsupported `tikv-importer.backend` (%s)", cfg.TikvImporter.Backend) } @@ -578,42 +549,8 @@ func (cfg *Config) Adjust(ctx context.Context) error { } if cfg.TikvImporter.Backend == BackendLocal { - if len(cfg.TikvImporter.SortedKVDir) == 0 { - return errors.Errorf("tikv-importer.sorted-kv-dir must not be empty!") - } - - storageSizeDir := filepath.Clean(cfg.TikvImporter.SortedKVDir) - sortedKVDirInfo, err := os.Stat(storageSizeDir) - switch { - case os.IsNotExist(err): - // the sorted-kv-dir does not exist, meaning we will create it automatically. - // so we extract the storage size from its parent directory. - storageSizeDir = filepath.Dir(storageSizeDir) - case err == nil: - if !sortedKVDirInfo.IsDir() { - return errors.Errorf("tikv-importer.sorted-kv-dir ('%s') is not a directory", storageSizeDir) - } - default: - return errors.Annotate(err, "invalid tikv-importer.sorted-kv-dir") - } - - if cfg.TikvImporter.DiskQuota == 0 { - enginesCount := uint64(cfg.App.IndexConcurrency + cfg.App.TableConcurrency) - writeAmount := uint64(cfg.App.RegionConcurrency) * uint64(cfg.Cron.CheckDiskQuota.Milliseconds()) - reservedSize := enginesCount*uint64(cfg.TikvImporter.EngineMemCacheSize) + writeAmount*autoDiskQuotaLocalReservedSpeed - - storageSize, err := common.GetStorageSize(storageSizeDir) - if err != nil { - return err - } - if storageSize.Available <= reservedSize { - return errors.Errorf( - "insufficient disk free space on `%s` (only %s, expecting >%s), please use a storage with enough free space, or specify `tikv-importer.disk-quota`", - cfg.TikvImporter.SortedKVDir, - units.BytesSize(float64(storageSize.Available)), - units.BytesSize(float64(reservedSize))) - } - cfg.TikvImporter.DiskQuota = ByteSize(storageSize.Available - reservedSize) + if err := cfg.CheckAndAdjustForLocalBackend(); err != nil { + return err } } @@ -632,25 +569,8 @@ func (cfg *Config) Adjust(ctx context.Context) error { return errors.Annotate(err, "invalid config: `mydumper.tidb.sql_mode` must be a valid SQL_MODE") } - if cfg.TiDB.Security == nil { - cfg.TiDB.Security = &cfg.Security - } - - switch cfg.TiDB.TLS { - case "": - if len(cfg.TiDB.Security.CAPath) > 0 { - cfg.TiDB.TLS = "cluster" - } else { - cfg.TiDB.TLS = "false" - } - case "cluster": - if len(cfg.Security.CAPath) == 0 { - return errors.New("invalid config: cannot set `tidb.tls` to 'cluster' without a [security] section") - } - case "false", "skip-verify", "preferred": - break - default: - return errors.Errorf("invalid config: unsupported `tidb.tls` config %s", cfg.TiDB.TLS) + if err := cfg.CheckAndAdjustSecurity(); err != nil { + return err } // mydumper.filter and black-white-list cannot co-exist. @@ -670,6 +590,92 @@ func (cfg *Config) Adjust(ctx context.Context) error { } } + if err := cfg.CheckAndAdjustTiDBPort(ctx, mustHaveInternalConnections); err != nil { + return err + } + cfg.AdjustMydumper() + cfg.AdjustCheckPoint() + return cfg.CheckAndAdjustFilePath() +} + +func (cfg *Config) CheckAndAdjustForLocalBackend() error { + if len(cfg.TikvImporter.SortedKVDir) == 0 { + return errors.Errorf("tikv-importer.sorted-kv-dir must not be empty!") + } + + storageSizeDir := filepath.Clean(cfg.TikvImporter.SortedKVDir) + sortedKVDirInfo, err := os.Stat(storageSizeDir) + switch { + case os.IsNotExist(err): + // the sorted-kv-dir does not exist, meaning we will create it automatically. + // so we extract the storage size from its parent directory. + storageSizeDir = filepath.Dir(storageSizeDir) + case err == nil: + if !sortedKVDirInfo.IsDir() { + return errors.Errorf("tikv-importer.sorted-kv-dir ('%s') is not a directory", storageSizeDir) + } + default: + return errors.Annotate(err, "invalid tikv-importer.sorted-kv-dir") + } + + if cfg.TikvImporter.DiskQuota == 0 { + enginesCount := uint64(cfg.App.IndexConcurrency + cfg.App.TableConcurrency) + writeAmount := uint64(cfg.App.RegionConcurrency) * uint64(cfg.Cron.CheckDiskQuota.Milliseconds()) + reservedSize := enginesCount*uint64(cfg.TikvImporter.EngineMemCacheSize) + writeAmount*autoDiskQuotaLocalReservedSpeed + + storageSize, err := common.GetStorageSize(storageSizeDir) + if err != nil { + return errors.Trace(err) + } + if storageSize.Available <= reservedSize { + return errors.Errorf( + "insufficient disk free space on `%s` (only %s, expecting >%s), please use a storage with enough free space, or specify `tikv-importer.disk-quota`", + cfg.TikvImporter.SortedKVDir, + units.BytesSize(float64(storageSize.Available)), + units.BytesSize(float64(reservedSize))) + } + cfg.TikvImporter.DiskQuota = ByteSize(storageSize.Available - reservedSize) + } + return nil +} + +func (cfg *Config) DefaultVarsForTiDBBackend() { + if cfg.App.IndexConcurrency == 0 { + cfg.App.IndexConcurrency = cfg.App.RegionConcurrency + } + if cfg.App.TableConcurrency == 0 { + cfg.App.TableConcurrency = cfg.App.RegionConcurrency + } +} + +func (cfg *Config) DefaultVarsForImporterAndLocalBackend() { + if cfg.App.IndexConcurrency == 0 { + cfg.App.IndexConcurrency = 2 + } + if cfg.App.TableConcurrency == 0 { + cfg.App.TableConcurrency = 6 + } + if cfg.TikvImporter.RangeConcurrency == 0 { + cfg.TikvImporter.RangeConcurrency = 16 + } + if cfg.TikvImporter.RegionSplitSize == 0 { + cfg.TikvImporter.RegionSplitSize = SplitRegionSize + } + if cfg.TiDB.DistSQLScanConcurrency == 0 { + cfg.TiDB.DistSQLScanConcurrency = defaultDistSQLScanConcurrency + } + if cfg.TiDB.BuildStatsConcurrency == 0 { + cfg.TiDB.BuildStatsConcurrency = defaultBuildStatsConcurrency + } + if cfg.TiDB.IndexSerialScanConcurrency == 0 { + cfg.TiDB.IndexSerialScanConcurrency = defaultIndexSerialScanConcurrency + } + if cfg.TiDB.ChecksumTableConcurrency == 0 { + cfg.TiDB.ChecksumTableConcurrency = defaultChecksumTableConcurrency + } +} + +func (cfg *Config) CheckAndAdjustTiDBPort(ctx context.Context, mustHaveInternalConnections bool) error { // automatically determine the TiDB port & PD address from TiDB settings if mustHaveInternalConnections && (cfg.TiDB.Port <= 0 || len(cfg.TiDB.PdAddr) == 0) { tls, err := cfg.ToTLS() @@ -697,47 +703,10 @@ func (cfg *Config) Adjust(ctx context.Context) error { if mustHaveInternalConnections && len(cfg.TiDB.PdAddr) == 0 { return errors.New("invalid `tidb.pd-addr` setting") } + return nil +} - // handle mydumper - if cfg.Mydumper.BatchSize <= 0 { - // if rows in source files are not sorted by primary key(if primary is number or cluster index enabled), - // the key range in each data engine may have overlap, thus a bigger engine size can somewhat alleviate it. - cfg.Mydumper.BatchSize = defaultBatchSize - } - if cfg.Mydumper.BatchImportRatio < 0.0 || cfg.Mydumper.BatchImportRatio >= 1.0 { - cfg.Mydumper.BatchImportRatio = 0.75 - } - if cfg.Mydumper.ReadBlockSize <= 0 { - cfg.Mydumper.ReadBlockSize = ReadBlockSize - } - if len(cfg.Mydumper.CharacterSet) == 0 { - cfg.Mydumper.CharacterSet = "auto" - } - - if len(cfg.Checkpoint.Schema) == 0 { - cfg.Checkpoint.Schema = "tidb_lightning_checkpoint" - } - if len(cfg.Checkpoint.Driver) == 0 { - cfg.Checkpoint.Driver = CheckpointDriverFile - } - if len(cfg.Checkpoint.DSN) == 0 { - switch cfg.Checkpoint.Driver { - case CheckpointDriverMySQL: - param := common.MySQLConnectParam{ - Host: cfg.TiDB.Host, - Port: cfg.TiDB.Port, - User: cfg.TiDB.User, - Password: cfg.TiDB.Psw, - SQLMode: mysql.DefaultSQLMode, - MaxAllowedPacket: defaultMaxAllowedPacket, - TLS: cfg.TiDB.TLS, - } - cfg.Checkpoint.DSN = param.ToDSN() - case CheckpointDriverFile: - cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb" - } - } - +func (cfg *Config) CheckAndAdjustFilePath() error { var u *url.URL // An absolute Windows path like "C:\Users\XYZ" would be interpreted as @@ -749,6 +718,7 @@ func (cfg *Config) Adjust(ctx context.Context) error { // On Windows, the drive letter can only be single letters from "A:" to "Z:", // so this won't mistake "S3:" as a Windows path. if len(filepath.VolumeName(cfg.Mydumper.SourceDir)) == 0 { + var err error u, err = url.Parse(cfg.Mydumper.SourceDir) if err != nil { return errors.Trace(err) @@ -781,7 +751,73 @@ func (cfg *Config) Adjust(ctx context.Context) error { if !found { return errors.Errorf("Unsupported data-source-dir url '%s'", cfg.Mydumper.SourceDir) } + return nil +} + +func (cfg *Config) AdjustCheckPoint() { + if len(cfg.Checkpoint.Schema) == 0 { + cfg.Checkpoint.Schema = "tidb_lightning_checkpoint" + } + if len(cfg.Checkpoint.Driver) == 0 { + cfg.Checkpoint.Driver = CheckpointDriverFile + } + if len(cfg.Checkpoint.DSN) == 0 { + switch cfg.Checkpoint.Driver { + case CheckpointDriverMySQL: + param := common.MySQLConnectParam{ + Host: cfg.TiDB.Host, + Port: cfg.TiDB.Port, + User: cfg.TiDB.User, + Password: cfg.TiDB.Psw, + SQLMode: mysql.DefaultSQLMode, + MaxAllowedPacket: defaultMaxAllowedPacket, + TLS: cfg.TiDB.TLS, + } + cfg.Checkpoint.DSN = param.ToDSN() + case CheckpointDriverFile: + cfg.Checkpoint.DSN = "/tmp/" + cfg.Checkpoint.Schema + ".pb" + } + } +} +func (cfg *Config) AdjustMydumper() { + if cfg.Mydumper.BatchSize <= 0 { + // if rows in source files are not sorted by primary key(if primary is number or cluster index enabled), + // the key range in each data engine may have overlap, thus a bigger engine size can somewhat alleviate it. + cfg.Mydumper.BatchSize = defaultBatchSize + } + if cfg.Mydumper.BatchImportRatio < 0.0 || cfg.Mydumper.BatchImportRatio >= 1.0 { + cfg.Mydumper.BatchImportRatio = 0.75 + } + if cfg.Mydumper.ReadBlockSize <= 0 { + cfg.Mydumper.ReadBlockSize = ReadBlockSize + } + if len(cfg.Mydumper.CharacterSet) == 0 { + cfg.Mydumper.CharacterSet = "auto" + } +} + +func (cfg *Config) CheckAndAdjustSecurity() error { + if cfg.TiDB.Security == nil { + cfg.TiDB.Security = &cfg.Security + } + + switch cfg.TiDB.TLS { + case "": + if len(cfg.TiDB.Security.CAPath) > 0 { + cfg.TiDB.TLS = "cluster" + } else { + cfg.TiDB.TLS = "false" + } + case "cluster": + if len(cfg.Security.CAPath) == 0 { + return errors.New("invalid config: cannot set `tidb.tls` to 'cluster' without a [security] section") + } + case "false", "skip-verify", "preferred": + break + default: + return errors.Errorf("invalid config: unsupported `tidb.tls` config %s", cfg.TiDB.TLS) + } return nil } diff --git a/pkg/lightning/config/configlist.go b/pkg/lightning/config/configlist.go index c78e8903b..00ff6c79b 100644 --- a/pkg/lightning/config/configlist.go +++ b/pkg/lightning/config/configlist.go @@ -20,14 +20,14 @@ import ( "time" ) -// ConfigList is a goroutine-safe FIFO list of *Config, which supports removal +// List is a goroutine-safe FIFO list of *Config, which supports removal // from the middle. The list is not expected to be very long. -type ConfigList struct { +type List struct { cond *sync.Cond taskIDMap map[int64]*list.Element nodes list.List - // lastID records the largest task ID being Push()'ed to the ConfigList. + // lastID records the largest task ID being Push()'ed to the List. // In the rare case where two Push() are executed in the same nanosecond // (or the not-so-rare case where the clock's precision is lower than CPU // speed), we'll need to manually force one of the task to use the ID as @@ -36,8 +36,8 @@ type ConfigList struct { } // NewConfigList creates a new ConfigList instance. -func NewConfigList() *ConfigList { - return &ConfigList{ +func NewConfigList() *List { + return &List{ cond: sync.NewCond(new(sync.Mutex)), taskIDMap: make(map[int64]*list.Element), } @@ -45,7 +45,7 @@ func NewConfigList() *ConfigList { // Push adds a configuration to the end of the list. The field `cfg.TaskID` will // be modified to include a unique ID to identify this task. -func (cl *ConfigList) Push(cfg *Config) { +func (cl *List) Push(cfg *Config) { id := time.Now().UnixNano() cl.cond.L.Lock() defer cl.cond.L.Unlock() @@ -63,7 +63,7 @@ func (cl *ConfigList) Push(cfg *Config) { // input context expired. // // If the context expired, the error field will contain the error from context. -func (cl *ConfigList) Pop(ctx context.Context) (*Config, error) { +func (cl *List) Pop(ctx context.Context) (*Config, error) { res := make(chan *Config) go func() { @@ -91,7 +91,7 @@ func (cl *ConfigList) Pop(ctx context.Context) (*Config, error) { // Remove removes a task from the list given its task ID. Returns true if a task // is successfully removed, false if the task ID did not exist. -func (cl *ConfigList) Remove(taskID int64) bool { +func (cl *List) Remove(taskID int64) bool { cl.cond.L.Lock() defer cl.cond.L.Unlock() element, ok := cl.taskIDMap[taskID] @@ -105,7 +105,7 @@ func (cl *ConfigList) Remove(taskID int64) bool { // Get obtains a task from the list given its task ID. If the task ID did not // exist, the returned bool field will be false. -func (cl *ConfigList) Get(taskID int64) (*Config, bool) { +func (cl *List) Get(taskID int64) (*Config, bool) { cl.cond.L.Lock() defer cl.cond.L.Unlock() element, ok := cl.taskIDMap[taskID] @@ -116,7 +116,7 @@ func (cl *ConfigList) Get(taskID int64) (*Config, bool) { } // AllIDs returns a list of all task IDs in the list. -func (cl *ConfigList) AllIDs() []int64 { +func (cl *List) AllIDs() []int64 { cl.cond.L.Lock() defer cl.cond.L.Unlock() res := make([]int64, 0, len(cl.taskIDMap)) @@ -128,7 +128,7 @@ func (cl *ConfigList) AllIDs() []int64 { // MoveToFront moves a task to the front of the list. Returns true if the task // is successfully moved (including no-op), false if the task ID did not exist. -func (cl *ConfigList) MoveToFront(taskID int64) bool { +func (cl *List) MoveToFront(taskID int64) bool { cl.cond.L.Lock() defer cl.cond.L.Unlock() element, ok := cl.taskIDMap[taskID] @@ -141,7 +141,7 @@ func (cl *ConfigList) MoveToFront(taskID int64) bool { // MoveToBack moves a task to the back of the list. Returns true if the task is // successfully moved (including no-op), false if the task ID did not exist. -func (cl *ConfigList) MoveToBack(taskID int64) bool { +func (cl *List) MoveToBack(taskID int64) bool { cl.cond.L.Lock() defer cl.cond.L.Unlock() element, ok := cl.taskIDMap[taskID] diff --git a/pkg/lightning/glue/glue.go b/pkg/lightning/glue/glue.go index cf3a17f5d..1bb1b1c91 100644 --- a/pkg/lightning/glue/glue.go +++ b/pkg/lightning/glue/glue.go @@ -38,7 +38,7 @@ type Glue interface { GetParser() *parser.Parser GetTables(context.Context, string) ([]*model.TableInfo, error) GetSession(context.Context) (checkpoints.Session, error) - OpenCheckpointsDB(context.Context, *config.Config) (checkpoints.CheckpointsDB, error) + OpenCheckpointsDB(context.Context, *config.Config) (checkpoints.DB, error) // Record is used to report some information (key, value) to host TiDB, including progress, stage currently Record(string, uint64) } @@ -170,7 +170,7 @@ func (e *ExternalTiDBGlue) GetSession(ctx context.Context) (checkpoints.Session, return &sqlConnSession{conn: conn}, nil } -func (e *ExternalTiDBGlue) OpenCheckpointsDB(ctx context.Context, cfg *config.Config) (checkpoints.CheckpointsDB, error) { +func (e *ExternalTiDBGlue) OpenCheckpointsDB(ctx context.Context, cfg *config.Config) (checkpoints.DB, error) { return checkpoints.OpenCheckpointsDB(ctx, cfg) } diff --git a/pkg/lightning/lightning.go b/pkg/lightning/lightning.go index 3575c9eec..3cca5d526 100755 --- a/pkg/lightning/lightning.go +++ b/pkg/lightning/lightning.go @@ -55,7 +55,7 @@ type Lightning struct { globalCfg *config.GlobalConfig globalTLS *common.TLS // taskCfgs is the list of task configurations enqueued in the server mode - taskCfgs *config.ConfigList + taskCfgs *config.List ctx context.Context shutdown context.CancelFunc // for whole lightning context server http.Server @@ -254,7 +254,9 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, g glue. return } taskCfg.TiDB.Security.CAPath = "" - taskCfg.TiDB.Security.RegisterMySQL() + if err := taskCfg.TiDB.Security.RegisterMySQL(); err != nil { + log.L().Warn("failed to deregister TLS config", log.ShortError(err)) + } }() // initiation of default glue should be after RegisterMySQL, which is ready to be called after taskCfg.Adjust @@ -302,7 +304,7 @@ func (l *Lightning) run(taskCtx context.Context, taskCfg *config.Config, g glue. dbMetas := mdl.GetDatabases() web.BroadcastInitProgress(dbMetas) - var procedure *restore.RestoreController + var procedure *restore.Controller procedure, err = restore.NewRestoreController(ctx, dbMetas, taskCfg, s, g) if err != nil { log.L().Error("restore failed", log.ShortError(err)) @@ -336,7 +338,7 @@ func writeJSONError(w http.ResponseWriter, code int, prefix string, err error) { if err != nil { prefix += ": " + err.Error() } - json.NewEncoder(w).Encode(errorResponse{Error: prefix}) + _ = json.NewEncoder(w).Encode(errorResponse{Error: prefix}) } func parseTaskID(req *http.Request) (int64, string, error) { @@ -362,6 +364,11 @@ func (l *Lightning) handleTask(w http.ResponseWriter, req *http.Request) { switch req.Method { case http.MethodGet: taskID, _, err := parseTaskID(req) + // golint tells us to refactor this with switch stmt. + // However switch stmt doesn't support init-statements, + // hence if we follow it things might be worse. + // Anyway, this chain of if-else isn't unacceptable. + //nolint:gocritic if e, ok := err.(*strconv.NumError); ok && e.Num == "" { l.handleGetTask(w) } else if err == nil { @@ -401,7 +408,7 @@ func (l *Lightning) handleGetTask(w http.ResponseWriter) { l.cancelLock.Unlock() w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(response) + _ = json.NewEncoder(w).Encode(response) } func (l *Lightning) handleGetOneTask(w http.ResponseWriter, req *http.Request, taskID int64) { @@ -468,7 +475,7 @@ func (l *Lightning) handlePostTask(w http.ResponseWriter, req *http.Request) { l.taskCfgs.Push(cfg) w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(taskResponse{ID: cfg.TaskID}) + _ = json.NewEncoder(w).Encode(taskResponse{ID: cfg.TaskID}) } func (l *Lightning) handleDeleteOneTask(w http.ResponseWriter, req *http.Request) { @@ -501,7 +508,7 @@ func (l *Lightning) handleDeleteOneTask(w http.ResponseWriter, req *http.Request if cancelSuccess { w.WriteHeader(http.StatusOK) - w.Write([]byte("{}")) + _, _ = w.Write([]byte("{}")) } else { writeJSONError(w, http.StatusNotFound, "task ID not found", nil) } @@ -532,7 +539,7 @@ func (l *Lightning) handlePatchOneTask(w http.ResponseWriter, req *http.Request) if moveSuccess { w.WriteHeader(http.StatusOK) - w.Write([]byte("{}")) + _, _ = w.Write([]byte("{}")) } else { writeJSONError(w, http.StatusNotFound, "task ID not found", nil) } @@ -540,15 +547,15 @@ func (l *Lightning) handlePatchOneTask(w http.ResponseWriter, req *http.Request) func writeBytesCompressed(w http.ResponseWriter, req *http.Request, b []byte) { if !strings.Contains(req.Header.Get("Accept-Encoding"), "gzip") { - w.Write(b) + _, _ = w.Write(b) return } w.Header().Set("Content-Encoding", "gzip") w.WriteHeader(http.StatusOK) gw, _ := gzip.NewWriterLevel(w, gzip.BestSpeed) - gw.Write(b) - gw.Close() + _, _ = gw.Write(b) + _ = gw.Close() } func handleProgressTask(w http.ResponseWriter, req *http.Request) { @@ -558,7 +565,7 @@ func handleProgressTask(w http.ResponseWriter, req *http.Request) { writeBytesCompressed(w, req, res) } else { w.WriteHeader(http.StatusInternalServerError) - json.NewEncoder(w).Encode(err.Error()) + _ = json.NewEncoder(w).Encode(err.Error()) } } @@ -574,7 +581,7 @@ func handleProgressTable(w http.ResponseWriter, req *http.Request) { } else { w.WriteHeader(http.StatusInternalServerError) } - json.NewEncoder(w).Encode(err.Error()) + _ = json.NewEncoder(w).Encode(err.Error()) } } @@ -590,7 +597,7 @@ func handlePause(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) restore.DeliverPauser.Pause() log.L().Info("progress paused") - w.Write([]byte("{}")) + _, _ = w.Write([]byte("{}")) default: w.Header().Set("Allow", http.MethodGet+", "+http.MethodPut) @@ -606,7 +613,7 @@ func handleResume(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) restore.DeliverPauser.Resume() log.L().Info("progress resumed") - w.Write([]byte("{}")) + _, _ = w.Write([]byte("{}")) default: w.Header().Set("Allow", http.MethodPut) @@ -625,7 +632,7 @@ func handleLogLevel(w http.ResponseWriter, req *http.Request) { case http.MethodGet: logLevel.Level = log.Level() w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(logLevel) + _ = json.NewEncoder(w).Encode(logLevel) case http.MethodPut, http.MethodPost: if err := json.NewDecoder(req.Body).Decode(&logLevel); err != nil { @@ -636,7 +643,7 @@ func handleLogLevel(w http.ResponseWriter, req *http.Request) { log.L().Info("changed log level", zap.Stringer("old", oldLevel), zap.Stringer("new", logLevel.Level)) log.SetLevel(logLevel.Level) w.WriteHeader(http.StatusOK) - w.Write([]byte("{}")) + _, _ = w.Write([]byte("{}")) default: w.Header().Set("Allow", http.MethodGet+", "+http.MethodPut+", "+http.MethodPost) @@ -678,7 +685,7 @@ func checkSystemRequirement(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta return nil } -/// checkSchemaConflict return error if checkpoint table scheme is conflict with data files +// checkSchemaConflict return error if checkpoint table scheme is conflict with data files func checkSchemaConflict(cfg *config.Config, dbsMeta []*mydump.MDDatabaseMeta) error { if cfg.Checkpoint.Enable && cfg.Checkpoint.Driver == config.CheckpointDriverMySQL { for _, db := range dbsMeta { diff --git a/pkg/lightning/lightning_test.go b/pkg/lightning/lightning_test.go index 897422aef..c3fa65535 100644 --- a/pkg/lightning/lightning_test.go +++ b/pkg/lightning/lightning_test.go @@ -11,6 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Contexts for HTTP requests communicating with a real HTTP server are essential, +// however, when the subject is a mocked server, it would probably be redundant. +//nolint:noctx package lightning import ( @@ -116,13 +119,13 @@ func (s *lightningServerSuite) SetUpTest(c *C) { s.lightning = New(cfg) s.taskCfgCh = make(chan *config.Config) s.lightning.ctx = context.WithValue(s.lightning.ctx, &taskCfgRecorderKey, s.taskCfgCh) - s.lightning.GoServe() + _ = s.lightning.GoServe() - failpoint.Enable("github.com/pingcap/br/pkg/lightning/SkipRunTask", "return") + _ = failpoint.Enable("github.com/pingcap/br/pkg/lightning/SkipRunTask", "return") } func (s *lightningServerSuite) TearDownTest(c *C) { - failpoint.Disable("github.com/pingcap/br/pkg/lightning/SkipRunTask") + _ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/SkipRunTask") s.lightning.Stop() } @@ -139,7 +142,10 @@ func (s *lightningServerSuite) TestRunServer(c *C) { c.Assert(data["error"], Equals, "server-mode not enabled") resp.Body.Close() - go s.lightning.RunServer() + go func() { + err := s.lightning.RunServer() + c.Assert(err, IsNil) + }() time.Sleep(100 * time.Millisecond) req, err := http.NewRequest(http.MethodPut, url, nil) @@ -226,7 +232,10 @@ func (s *lightningServerSuite) TestGetDeleteTask(c *C) { return result.ID } - go s.lightning.RunServer() + go func() { + err := s.lightning.RunServer() + c.Assert(err, IsNil) + }() time.Sleep(100 * time.Millisecond) // Check `GET /tasks` without any active tasks @@ -415,7 +424,7 @@ func (s *lightningServerSuite) TestHTTPAPIOutsideServerMode(c *C) { resp, err = http.DefaultClient.Do(req) c.Assert(err, IsNil) c.Assert(resp.StatusCode, Equals, http.StatusOK) - + c.Assert(resp.Body.Close(), IsNil) // ... and the task should be canceled now. c.Assert(<-errCh, Equals, context.Canceled) } @@ -473,7 +482,13 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) { c.Assert(err, IsNil) err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/SetRlimitError", "return(true)") c.Assert(err, IsNil) +<<<<<<< HEAD defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/SetRlimitError") +======= + defer func() { + _ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/SetRlimitError") + }() +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) // with this dbMetas, the estimated fds will be 2050, so should return error err = checkSystemRequirement(cfg, dbMetas) c.Assert(err, NotNil) @@ -488,8 +503,15 @@ func (s *lightningServerSuite) TestCheckSystemRequirement(c *C) { c.Assert(err, IsNil) // the min rlimit should be bigger than the default min value (16384) +<<<<<<< HEAD err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/GetRlimitValue", "return(8200)") defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/GetRlimitValue") +======= + err = failpoint.Enable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue", "return(8200)") + defer func() { + _ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/backend/local/GetRlimitValue") + }() +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) c.Assert(err, IsNil) err = checkSystemRequirement(cfg, dbMetas) c.Assert(err, IsNil) diff --git a/pkg/lightning/log/log.go b/pkg/lightning/log/log.go index 8ed79151f..50d938412 100644 --- a/pkg/lightning/log/log.go +++ b/pkg/lightning/log/log.go @@ -75,7 +75,9 @@ var ( // InitLogger initializes Lightning's and also the TiDB library's loggers. func InitLogger(cfg *Config, tidbLoglevel string) error { - logutil.InitLogger(&logutil.LogConfig{Config: pclog.Config{Level: tidbLoglevel}}) + if err := logutil.InitLogger(&logutil.LogConfig{Config: pclog.Config{Level: tidbLoglevel}}); err != nil { + return errors.Trace(err) + } logCfg := &pclog.Config{ Level: cfg.Level, diff --git a/pkg/lightning/metric/metric.go b/pkg/lightning/metric/metric.go index b4ffdc9c5..ef611828e 100644 --- a/pkg/lightning/metric/metric.go +++ b/pkg/lightning/metric/metric.go @@ -197,6 +197,7 @@ var ( ) ) +//nolint:gochecknoinits // TODO: refactor func init() { prometheus.MustRegister(IdleWorkersGauge) prometheus.MustRegister(ImporterEngineCounter) diff --git a/pkg/lightning/mock/glue.go b/pkg/lightning/mock/glue.go index 63ece855e..875dfa6e8 100644 --- a/pkg/lightning/mock/glue.go +++ b/pkg/lightning/mock/glue.go @@ -128,10 +128,10 @@ func (mr *MockGlueMockRecorder) GetSession(arg0 interface{}) *gomock.Call { } // OpenCheckpointsDB mocks base method -func (m *MockGlue) OpenCheckpointsDB(arg0 context.Context, arg1 *config.Config) (checkpoints.CheckpointsDB, error) { +func (m *MockGlue) OpenCheckpointsDB(arg0 context.Context, arg1 *config.Config) (checkpoints.DB, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "OpenCheckpointsDB", arg0, arg1) - ret0, _ := ret[0].(checkpoints.CheckpointsDB) + ret0, _ := ret[0].(checkpoints.DB) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/lightning/mydump/csv_parser.go b/pkg/lightning/mydump/csv_parser.go index 1e3db9dd3..7ff08f09f 100644 --- a/pkg/lightning/mydump/csv_parser.go +++ b/pkg/lightning/mydump/csv_parser.go @@ -17,7 +17,6 @@ import ( "bytes" "io" "strings" - "unicode" "github.com/pingcap/br/pkg/utils" @@ -37,8 +36,7 @@ var ( // CSVParser is basically a copy of encoding/csv, but special-cased for MySQL-like input. type CSVParser struct { blockParser - cfg *config.CSVConfig - escFlavor backslashEscapeFlavor + cfg *config.CSVConfig comma []byte quote []byte @@ -57,6 +55,7 @@ type CSVParser struct { lastRecord []string + escFlavor backslashEscapeFlavor // if set to true, csv parser will treat the first non-empty line as header line shouldParseHeader bool } @@ -131,6 +130,7 @@ func (parser *CSVParser) readByte() (byte, error) { return b, nil } +//nolint:unused // Maybe useful in the future. func (parser *CSVParser) readBytes(buf []byte) (int, error) { cnt := 0 for cnt < len(buf) { @@ -290,7 +290,7 @@ outside: continue } // skip lines only contain whitespaces - if err == nil && whitespaceLine && len(bytes.TrimFunc(parser.recordBuffer, unicode.IsSpace)) == 0 { + if err == nil && whitespaceLine && len(bytes.TrimSpace(parser.recordBuffer)) == 0 { parser.recordBuffer = parser.recordBuffer[:0] continue } @@ -391,9 +391,8 @@ func (parser *CSVParser) readQuotedField() error { if !bytes.Equal(b, parser.quote) { if parser.quote[0] == parser.comma[0] { return processComma() - } else { - return processDefault() } + return processDefault() } } parser.skipBytes(len(parser.quote)) @@ -567,10 +566,10 @@ func (parser *CSVParser) ReadColumns() error { return nil } -var newLineAsciiSet = makeByteSet([]byte{'\r', '\n'}) +var newLineASCIISet = makeByteSet([]byte{'\r', '\n'}) func indexOfNewLine(b []byte) int { - return IndexAnyByte(b, &newLineAsciiSet) + return IndexAnyByte(b, &newLineASCIISet) } func (parser *CSVParser) ReadUntilTokNewLine() (int64, error) { diff --git a/pkg/lightning/mydump/loader.go b/pkg/lightning/mydump/loader.go index 73f2f40e8..eb48d2222 100644 --- a/pkg/lightning/mydump/loader.go +++ b/pkg/lightning/mydump/loader.go @@ -91,11 +91,11 @@ type mdLoaderSetup struct { func NewMyDumpLoader(ctx context.Context, cfg *config.Config) (*MDLoader, error) { u, err := storage.ParseBackend(cfg.Mydumper.SourceDir, nil) if err != nil { - return nil, err + return nil, errors.Trace(err) } s, err := storage.New(ctx, u, &storage.ExternalStorageOptions{SkipCheckPath: true}) if err != nil { - return nil, err + return nil, errors.Trace(err) } return NewMyDumpLoaderWithStore(ctx, cfg, s) @@ -219,7 +219,7 @@ func (s *mdLoaderSetup) setup(ctx context.Context, store storage.ExternalStorage if !s.loader.noSchema { // setup database schema if len(s.dbSchemas) == 0 { - return errors.New("no schema create sql files found. Please either set `mydumper.no-schema` to true or add schema sql file for each database.") + return errors.New("no schema create sql files found. Please either set `mydumper.no-schema` to true or add schema sql file for each database") } for _, fileInfo := range s.dbSchemas { if _, dbExists := s.insertDB(fileInfo.TableName.Schema, fileInfo.FileMeta.Path); dbExists && s.loader.router == nil { @@ -267,12 +267,13 @@ func (s *mdLoaderSetup) setup(ctx context.Context, store storage.ExternalStorage for _, dbMeta := range s.loader.dbs { // Put the small table in the front of the slice which can avoid large table // take a long time to import and block small table to release index worker. - sort.SliceStable(dbMeta.Tables, func(i, j int) bool { - return dbMeta.Tables[i].TotalSize < dbMeta.Tables[j].TotalSize + meta := dbMeta + sort.SliceStable(meta.Tables, func(i, j int) bool { + return meta.Tables[i].TotalSize < meta.Tables[j].TotalSize }) // sort each table source files by sort-key - for _, tbMeta := range dbMeta.Tables { + for _, tbMeta := range meta.Tables { dataFiles := tbMeta.DataFiles sort.SliceStable(dataFiles, func(i, j int) bool { return dataFiles[i].FileMeta.SortKey < dataFiles[j].FileMeta.SortKey @@ -419,16 +420,15 @@ func (s *mdLoaderSetup) insertDB(dbName string, path string) (*MDDatabaseMeta, b dbIndex, ok := s.dbIndexMap[dbName] if ok { return s.loader.dbs[dbIndex], true - } else { - s.dbIndexMap[dbName] = len(s.loader.dbs) - ptr := &MDDatabaseMeta{ - Name: dbName, - SchemaFile: path, - charSet: s.loader.charSet, - } - s.loader.dbs = append(s.loader.dbs, ptr) - return ptr, false } + s.dbIndexMap[dbName] = len(s.loader.dbs) + ptr := &MDDatabaseMeta{ + Name: dbName, + SchemaFile: path, + charSet: s.loader.charSet, + } + s.loader.dbs = append(s.loader.dbs, ptr) + return ptr, false } func (s *mdLoaderSetup) insertTable(fileInfo FileInfo) (*MDTableMeta, bool, bool) { @@ -436,18 +436,17 @@ func (s *mdLoaderSetup) insertTable(fileInfo FileInfo) (*MDTableMeta, bool, bool tableIndex, ok := s.tableIndexMap[fileInfo.TableName] if ok { return dbMeta.Tables[tableIndex], dbExists, true - } else { - s.tableIndexMap[fileInfo.TableName] = len(dbMeta.Tables) - ptr := &MDTableMeta{ - DB: fileInfo.TableName.Schema, - Name: fileInfo.TableName.Name, - SchemaFile: fileInfo, - DataFiles: make([]FileInfo, 0, 16), - charSet: s.loader.charSet, - } - dbMeta.Tables = append(dbMeta.Tables, ptr) - return ptr, dbExists, false } + s.tableIndexMap[fileInfo.TableName] = len(dbMeta.Tables) + ptr := &MDTableMeta{ + DB: fileInfo.TableName.Schema, + Name: fileInfo.TableName.Name, + SchemaFile: fileInfo, + DataFiles: make([]FileInfo, 0, 16), + charSet: s.loader.charSet, + } + dbMeta.Tables = append(dbMeta.Tables, ptr) + return ptr, dbExists, false } func (s *mdLoaderSetup) insertView(fileInfo FileInfo) (bool, bool) { diff --git a/pkg/lightning/mydump/loader_test.go b/pkg/lightning/mydump/loader_test.go index 816e904d8..21c7e534f 100644 --- a/pkg/lightning/mydump/loader_test.go +++ b/pkg/lightning/mydump/loader_test.go @@ -58,14 +58,13 @@ func (s *testMydumpLoaderSuite) SetUpTest(c *C) { s.cfg = newConfigWithSourceDir(s.sourceDir) } -func (s *testMydumpLoaderSuite) touch(c *C, filename ...string) string { +func (s *testMydumpLoaderSuite) touch(c *C, filename ...string) { components := make([]string, len(filename)+1) components = append(components, s.sourceDir) components = append(components, filename...) path := filepath.Join(components...) err := ioutil.WriteFile(path, nil, 0o644) c.Assert(err, IsNil) - return path } func (s *testMydumpLoaderSuite) mkdir(c *C, dirname string) { @@ -108,7 +107,7 @@ func (s *testMydumpLoaderSuite) TestLoader(c *C) { func (s *testMydumpLoaderSuite) TestEmptyDB(c *C) { _, err := md.NewMyDumpLoader(context.Background(), s.cfg) - c.Assert(err, ErrorMatches, "no schema create sql files found. Please either set `mydumper.no-schema` to true or add schema sql file for each database.") + c.Assert(err, ErrorMatches, "no schema create sql files found. Please either set `mydumper.no-schema` to true or add schema sql file for each database") } func (s *testMydumpLoaderSuite) TestDuplicatedDB(c *C) { @@ -481,7 +480,7 @@ func (s *testMydumpLoaderSuite) TestFileRouting(c *C) { s.touch(c, "d1/test2.001.sql") s.touch(c, "d1/v1-table.sql") s.touch(c, "d1/v1-view.sql") - _ = s.touch(c, "d1/t1-schema-create.sql") + s.touch(c, "d1/t1-schema-create.sql") s.touch(c, "d2/schema.sql") s.touch(c, "d2/abc-table.sql") s.touch(c, "abc.1.sql") diff --git a/pkg/lightning/mydump/parser.go b/pkg/lightning/mydump/parser.go index e08d3e3ac..18079199e 100644 --- a/pkg/lightning/mydump/parser.go +++ b/pkg/lightning/mydump/parser.go @@ -276,8 +276,8 @@ func unescape( ) string { if len(delim) > 0 { delim2 := delim + delim - if strings.Index(input, delim2) != -1 { - input = strings.Replace(input, delim2, delim, -1) + if strings.Contains(input, delim2) { + input = strings.ReplaceAll(input, delim2, delim) } } if escFlavor != backslashEscapeFlavorNone && strings.IndexByte(input, '\\') != -1 { @@ -500,13 +500,13 @@ func (parser *ChunkParser) ReadRow() error { case tokHexString: hexLit, err := types.ParseHexStr(string(content)) if err != nil { - return err + return errors.Trace(err) } value.SetBinaryLiteral(hexLit) case tokBinString: binLit, err := types.ParseBitStr(string(content)) if err != nil { - return err + return errors.Trace(err) } value.SetBinaryLiteral(binLit) default: @@ -527,6 +527,9 @@ func (parser *blockParser) LastRow() Row { // RecycleRow places the row object back into the allocation pool. func (parser *blockParser) RecycleRow(row Row) { + // We need farther benchmarking to make sure whether send a pointer + // (instead of a slice) here can improve performance. + //nolint:staticcheck parser.rowPool.Put(row.Row[:0]) } diff --git a/pkg/lightning/mydump/reader_test.go b/pkg/lightning/mydump/reader_test.go index 7785c86b1..c66a3686e 100644 --- a/pkg/lightning/mydump/reader_test.go +++ b/pkg/lightning/mydump/reader_test.go @@ -27,8 +27,6 @@ import ( "github.com/pingcap/br/pkg/storage" ) -////////////////////////////////////////////////////////// - var _ = Suite(&testMydumpReaderSuite{}) type testMydumpReaderSuite struct{} @@ -59,12 +57,7 @@ func (s *testMydumpReaderSuite) TestExportStatementNoTrailingNewLine(c *C) { } func (s *testMydumpReaderSuite) TestExportStatementWithComment(c *C) { - dir := c.MkDir() - file, err := ioutil.TempFile(dir, "tidb_lightning_test_reader") - c.Assert(err, IsNil) - defer os.Remove(file.Name()) - - _, err = file.Write([]byte(` + s.exportStatmentShouldBe(c, ` /* whatever blabla multiple lines comment multiple lines comment @@ -73,29 +66,11 @@ func (s *testMydumpReaderSuite) TestExportStatementWithComment(c *C) { multiple lines comment */; CREATE DATABASE whatever; -`)) - c.Assert(err, IsNil) - stat, err := file.Stat() - c.Assert(err, IsNil) - err = file.Close() - c.Assert(err, IsNil) - - store, err := storage.NewLocalStorage(dir) - c.Assert(err, IsNil) - - f := FileInfo{FileMeta: SourceFileMeta{Path: stat.Name(), FileSize: stat.Size()}} - data, err := ExportStatement(context.TODO(), store, f, "auto") - c.Assert(err, IsNil) - c.Assert(data, DeepEquals, []byte("CREATE DATABASE whatever;")) +`, "CREATE DATABASE whatever;") } func (s *testMydumpReaderSuite) TestExportStatementWithCommentNoTrailingNewLine(c *C) { - dir := c.MkDir() - file, err := ioutil.TempFile(dir, "tidb_lightning_test_reader") - c.Assert(err, IsNil) - defer os.Remove(file.Name()) - - _, err = file.Write([]byte(` + s.exportStatmentShouldBe(c, ` /* whatever blabla multiple lines comment multiple lines comment @@ -103,7 +78,16 @@ func (s *testMydumpReaderSuite) TestExportStatementWithCommentNoTrailingNewLine( multiple lines comment multiple lines comment */; - CREATE DATABASE whatever;`)) + CREATE DATABASE whatever;`, "CREATE DATABASE whatever;") +} + +func (s *testMydumpReaderSuite) exportStatmentShouldBe(c *C, stmt string, expected string) { + dir := c.MkDir() + file, err := ioutil.TempFile(dir, "tidb_lightning_test_reader") + c.Assert(err, IsNil) + defer os.Remove(file.Name()) + + _, err = file.Write([]byte(stmt)) c.Assert(err, IsNil) stat, err := file.Stat() c.Assert(err, IsNil) @@ -115,7 +99,7 @@ func (s *testMydumpReaderSuite) TestExportStatementWithCommentNoTrailingNewLine( f := FileInfo{FileMeta: SourceFileMeta{Path: stat.Name(), FileSize: stat.Size()}} data, err := ExportStatement(context.TODO(), store, f, "auto") c.Assert(err, IsNil) - c.Assert(data, DeepEquals, []byte("CREATE DATABASE whatever;")) + c.Assert(data, DeepEquals, []byte(expected)) } func (s *testMydumpReaderSuite) TestExportStatementGBK(c *C) { @@ -169,11 +153,11 @@ func (s *testMydumpReaderSuite) TestExportStatementGibberishError(c *C) { type AlwaysErrorReadSeekCloser struct{} func (AlwaysErrorReadSeekCloser) Read([]byte) (int, error) { - return 0, errors.New("read error!") + return 0, errors.New("read error") } func (AlwaysErrorReadSeekCloser) Seek(int64, int) (int64, error) { - return 0, errors.New("seek error!") + return 0, errors.New("seek error") } func (AlwaysErrorReadSeekCloser) Close() error { @@ -193,5 +177,5 @@ func (s *testMydumpReaderSuite) TestExportStatementHandleNonEOFError(c *C) { f := FileInfo{FileMeta: SourceFileMeta{Path: "no-perm-file", FileSize: 1}} _, err := ExportStatement(ctx, mockStorage, f, "auto") - c.Assert(err, ErrorMatches, "read error!") + c.Assert(err, ErrorMatches, "read error") } diff --git a/pkg/lightning/mydump/region.go b/pkg/lightning/mydump/region.go index 9d7f1fb51..9b4634fba 100644 --- a/pkg/lightning/mydump/region.go +++ b/pkg/lightning/mydump/region.go @@ -58,8 +58,6 @@ func (reg *TableRegion) Size() int64 { return reg.Chunk.EndOffset - reg.Chunk.Offset } -//////////////////////////////////////////////////////////////// - func AllocateEngineIDs( filesRegions []*TableRegion, dataFileSizes []float64, @@ -262,7 +260,6 @@ func makeSourceFileRegion( // If a csv file is overlarge, we need to split it into multiple regions. // Note: We can only split a csv file whose format is strict. if isCsvFile && dataFileSize > int64(cfg.Mydumper.MaxRegionSize) && cfg.Mydumper.StrictFormat { - _, regions, subFileSizes, err := SplitLargeFile(ctx, meta, cfg, fi, divisor, 0, ioWorkers, store) return regions, subFileSizes, err } @@ -302,6 +299,9 @@ func makeParquetFileRegion( return prevRowIdxMax, nil, errors.Trace(err) } numberRows, err := ReadParquetFileRowCount(ctx, store, r, dataFile.FileMeta.Path) + if err != nil { + return 0, nil, errors.Trace(err) + } rowIDMax := prevRowIdxMax + numberRows region := &TableRegion{ DB: meta.DB, @@ -333,7 +333,7 @@ func SplitLargeFile( prevRowIdxMax int64, ioWorker *worker.Pool, store storage.ExternalStorage, -) (prevRowIdMax int64, regions []*TableRegion, dataFileSizes []float64, err error) { +) (prevRowIDMax int64, regions []*TableRegion, dataFileSizes []float64, err error) { maxRegionSize := int64(cfg.Mydumper.MaxRegionSize) dataFileSizes = make([]float64, 0, dataFile.FileMeta.FileSize/maxRegionSize+1) startOffset, endOffset := int64(0), maxRegionSize @@ -360,7 +360,7 @@ func SplitLargeFile( return 0, nil, nil, err } parser := NewCSVParser(&cfg.Mydumper.CSV, r, int64(cfg.Mydumper.ReadBlockSize), ioWorker, false) - if err = parser.SetPos(endOffset, prevRowIdMax); err != nil { + if err = parser.SetPos(endOffset, prevRowIDMax); err != nil { return 0, nil, nil, err } pos, err := parser.ReadUntilTokNewLine() diff --git a/pkg/lightning/mydump/region_test.go b/pkg/lightning/mydump/region_test.go index 4c0062721..e44915ce1 100644 --- a/pkg/lightning/mydump/region_test.go +++ b/pkg/lightning/mydump/region_test.go @@ -41,21 +41,6 @@ func (s *testMydumpRegionSuite) TearDownSuite(c *C) {} // "tbl_multi_index": 10000, // } -func getFileSize(file string) (int64, error) { - fd, err := os.Open(file) - if err != nil { - return -1, err - } - defer fd.Close() - - fstat, err := fd.Stat() - if err != nil { - return -1, err - } - - return fstat.Size(), nil -} - /* TODO : test with specified 'regionBlockSize' ... */ diff --git a/pkg/lightning/mydump/router.go b/pkg/lightning/mydump/router.go index 10b87e0b1..8ec5857a3 100644 --- a/pkg/lightning/mydump/router.go +++ b/pkg/lightning/mydump/router.go @@ -196,7 +196,6 @@ func (p regexRouterParser) Parse(r *config.FileRouteRule) (*RegexRouter, error) r.Type = quoteTmplFn(r.Type) r.Compression = quoteTmplFn(r.Compression) r.Key = quoteTmplFn(r.Key) - } pattern, err := regexp.Compile(r.Pattern) if err != nil { diff --git a/pkg/lightning/mydump/router_test.go b/pkg/lightning/mydump/router_test.go index e308bd842..8c5e2f178 100644 --- a/pkg/lightning/mydump/router_test.go +++ b/pkg/lightning/mydump/router_test.go @@ -145,7 +145,7 @@ func (t *testFileRouterSuite) TestMultiRouteRule(c *C) { "/test/123/my_schema.my_table.sql": {"my_schema", "my_table", "", "", "sql"}, "my_dir/my_schema.my_table.csv": {"my_schema", "my_table", "", "", "csv"}, "my_schema.my_table.0001.sql": {"my_schema", "my_table", "0001", "", "sql"}, - //"my_schema.my_table.0001.sql.gz": {"my_schema", "my_table", "0001", "gz", "sql"}, + // "my_schema.my_table.0001.sql.gz": {"my_schema", "my_table", "0001", "gz", "sql"}, } for path, fields := range inputOutputMap { res, err := r.Route(path) diff --git a/pkg/lightning/restore/checksum.go b/pkg/lightning/restore/checksum.go index 53395896b..0614ad2bc 100644 --- a/pkg/lightning/restore/checksum.go +++ b/pkg/lightning/restore/checksum.go @@ -21,7 +21,7 @@ import ( "go.uber.org/zap" "github.com/pingcap/br/pkg/checksum" - . "github.com/pingcap/br/pkg/lightning/checkpoints" + "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/log" @@ -52,10 +52,10 @@ type RemoteChecksum struct { } type ChecksumManager interface { - Checksum(ctx context.Context, tableInfo *TidbTableInfo) (*RemoteChecksum, error) + Checksum(ctx context.Context, tableInfo *checkpoints.TidbTableInfo) (*RemoteChecksum, error) } -func newChecksumManager(ctx context.Context, rc *RestoreController) (ChecksumManager, error) { +func newChecksumManager(ctx context.Context, rc *Controller) (ChecksumManager, error) { // if we don't need checksum, just return nil if rc.cfg.TikvImporter.Backend == config.BackendTiDB || rc.cfg.PostRestore.Checksum == config.OpLevelOff { return nil, nil @@ -114,7 +114,7 @@ func newTiDBChecksumExecutor(db *sql.DB) *tidbChecksumExecutor { } } -func (e *tidbChecksumExecutor) Checksum(ctx context.Context, tableInfo *TidbTableInfo) (*RemoteChecksum, error) { +func (e *tidbChecksumExecutor) Checksum(ctx context.Context, tableInfo *checkpoints.TidbTableInfo) (*RemoteChecksum, error) { var err error if err = e.manager.addOneJob(ctx, e.db); err != nil { return nil, err @@ -149,7 +149,7 @@ func (e *tidbChecksumExecutor) Checksum(ctx context.Context, tableInfo *TidbTabl // DoChecksum do checksum for tables. // table should be in ., format. e.g. foo.bar -func DoChecksum(ctx context.Context, table *TidbTableInfo) (*RemoteChecksum, error) { +func DoChecksum(ctx context.Context, table *checkpoints.TidbTableInfo) (*RemoteChecksum, error) { var err error manager, ok := ctx.Value(&checksumManagerKey).(ChecksumManager) if !ok { @@ -195,7 +195,7 @@ func (m *gcLifeTimeManager) addOneJob(ctx context.Context, db *sql.DB) error { return err } } - m.runningJobs += 1 + m.runningJobs++ return nil } @@ -207,7 +207,7 @@ func (m *gcLifeTimeManager) removeOneJob(ctx context.Context, db *sql.DB) { m.runningJobsLock.Lock() defer m.runningJobsLock.Unlock() - m.runningJobs -= 1 + m.runningJobs-- if m.runningJobs == 0 { err := UpdateGCLifeTime(ctx, db, m.oriGCLifeTime) if err != nil { @@ -266,7 +266,7 @@ func newTiKVChecksumManager(client kv.Client, pdClient pd.Client, distSQLScanCon } } -func (e *tikvChecksumManager) checksumDB(ctx context.Context, tableInfo *TidbTableInfo) (*RemoteChecksum, error) { +func (e *tikvChecksumManager) checksumDB(ctx context.Context, tableInfo *checkpoints.TidbTableInfo) (*RemoteChecksum, error) { physicalTS, logicalTS, err := e.manager.pdClient.GetTS(ctx) if err != nil { return nil, errors.Annotate(err, "fetch tso from pd failed") @@ -312,7 +312,7 @@ func (e *tikvChecksumManager) checksumDB(ctx context.Context, tableInfo *TidbTab return nil, err } -func (e *tikvChecksumManager) Checksum(ctx context.Context, tableInfo *TidbTableInfo) (*RemoteChecksum, error) { +func (e *tikvChecksumManager) Checksum(ctx context.Context, tableInfo *checkpoints.TidbTableInfo) (*RemoteChecksum, error) { tbl := common.UniqueTable(tableInfo.DB, tableInfo.Name) err := e.manager.addOneJob(ctx, tbl, oracle.ComposeTS(time.Now().Unix()*1000, 0)) if err != nil { @@ -356,7 +356,7 @@ type gcTTLManager struct { pdClient pd.Client // tableGCSafeTS is a binary heap that stored active checksum jobs GC safe point ts tableGCSafeTS []*tableChecksumTS - currentTs uint64 + currentTS uint64 serviceID string // 0 for not start, otherwise started started uint32 @@ -376,15 +376,15 @@ func (m *gcTTLManager) addOneJob(ctx context.Context, table string, ts uint64) e } m.lock.Lock() defer m.lock.Unlock() - var curTs uint64 + var curTS uint64 if len(m.tableGCSafeTS) > 0 { - curTs = m.tableGCSafeTS[0].gcSafeTS + curTS = m.tableGCSafeTS[0].gcSafeTS } m.Push(&tableChecksumTS{table: table, gcSafeTS: ts}) heap.Fix(m, len(m.tableGCSafeTS)-1) - m.currentTs = m.tableGCSafeTS[0].gcSafeTS - if curTs == 0 || m.currentTs < curTs { - return m.doUpdateGCTTL(ctx, m.currentTs) + m.currentTS = m.tableGCSafeTS[0].gcSafeTS + if curTS == 0 || m.currentTS < curTS { + return m.doUpdateGCTTL(ctx, m.currentTS) } return nil } @@ -409,18 +409,18 @@ func (m *gcTTLManager) removeOneJob(table string) { } } - var newTs uint64 + var newTS uint64 if len(m.tableGCSafeTS) > 0 { - newTs = m.tableGCSafeTS[0].gcSafeTS + newTS = m.tableGCSafeTS[0].gcSafeTS } - m.currentTs = newTs + m.currentTS = newTS } func (m *gcTTLManager) updateGCTTL(ctx context.Context) error { m.lock.Lock() - currentTs := m.currentTs + currentTS := m.currentTS m.lock.Unlock() - return m.doUpdateGCTTL(ctx, currentTs) + return m.doUpdateGCTTL(ctx, currentTS) } func (m *gcTTLManager) doUpdateGCTTL(ctx context.Context, ts uint64) error { diff --git a/pkg/lightning/restore/checksum_test.go b/pkg/lightning/restore/checksum_test.go index 25c73caa9..2ca86f537 100644 --- a/pkg/lightning/restore/checksum_test.go +++ b/pkg/lightning/restore/checksum_test.go @@ -176,7 +176,7 @@ func (s *checksumSuite) TestDoChecksumWithTikv(c *C) { kvClient.maxErrCount = i kvClient.curErrCount = 0 checksumExec := &tikvChecksumManager{manager: newGCTTLManager(pdClient), client: kvClient} - startTs := oracle.ComposeTS(time.Now().Unix()*1000, 0) + startTS := oracle.ComposeTS(time.Now().Unix()*1000, 0) ctx := context.WithValue(context.Background(), &checksumManagerKey, checksumExec) _, err = DoChecksum(ctx, &TidbTableInfo{DB: "test", Name: "t", Core: tableInfo}) // with max error retry < maxErrorRetryCount, the checksum can success @@ -190,7 +190,7 @@ func (s *checksumSuite) TestDoChecksumWithTikv(c *C) { // after checksum, safepint should be small than start ts ts := pdClient.currentSafePoint() // 1ms for the schedule deviation - c.Assert(ts <= startTs+1, IsTrue) + c.Assert(ts <= startTS+1, IsTrue) c.Assert(atomic.LoadUint32(&checksumExec.manager.started) > 0, IsTrue) } } @@ -311,23 +311,23 @@ func (s *checksumSuite) TestGcTTLManagerMulti(c *C) { for i := uint64(1); i <= 5; i++ { err := manager.addOneJob(ctx, fmt.Sprintf("test%d", i), i) c.Assert(err, IsNil) - c.Assert(manager.currentTs, Equals, uint64(1)) + c.Assert(manager.currentTS, Equals, uint64(1)) } manager.removeOneJob("test2") - c.Assert(manager.currentTs, Equals, uint64(1)) + c.Assert(manager.currentTS, Equals, uint64(1)) manager.removeOneJob("test1") - c.Assert(manager.currentTs, Equals, uint64(3)) + c.Assert(manager.currentTS, Equals, uint64(3)) manager.removeOneJob("test3") - c.Assert(manager.currentTs, Equals, uint64(4)) + c.Assert(manager.currentTS, Equals, uint64(4)) manager.removeOneJob("test4") - c.Assert(manager.currentTs, Equals, uint64(5)) + c.Assert(manager.currentTS, Equals, uint64(5)) manager.removeOneJob("test5") - c.Assert(manager.currentTs, Equals, uint64(0)) + c.Assert(manager.currentTS, Equals, uint64(0)) } func (s *checksumSuite) TestPdServiceID(c *C) { diff --git a/pkg/lightning/restore/const.go b/pkg/lightning/restore/const.go deleted file mode 100644 index 082940c7e..000000000 --- a/pkg/lightning/restore/const.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2019 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package restore - -const ( - defReadBlockSize int64 = 1024 * 128 // TODO ... config -) diff --git a/pkg/lightning/restore/restore.go b/pkg/lightning/restore/restore.go index 7e5e5be55..33801de66 100644 --- a/pkg/lightning/restore/restore.go +++ b/pkg/lightning/restore/restore.go @@ -39,8 +39,17 @@ import ( "go.uber.org/zap" "modernc.org/mathutil" +<<<<<<< HEAD kv "github.com/pingcap/br/pkg/lightning/backend" . "github.com/pingcap/br/pkg/lightning/checkpoints" +======= + "github.com/pingcap/br/pkg/lightning/backend" + "github.com/pingcap/br/pkg/lightning/backend/importer" + "github.com/pingcap/br/pkg/lightning/backend/kv" + "github.com/pingcap/br/pkg/lightning/backend/local" + "github.com/pingcap/br/pkg/lightning/backend/tidb" + "github.com/pingcap/br/pkg/lightning/checkpoints" +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/glue" @@ -77,8 +86,8 @@ const ( // DeliverPauser is a shared pauser to pause progress to (*chunkRestore).encodeLoop var DeliverPauser = common.NewPauser() +// nolint:gochecknoinits // TODO: refactor func init() { - // used in integration tests failpoint.Inject("SetMinDeliverBytes", func(v failpoint.Value) { minDeliverBytes = uint64(v.(int)) }) @@ -86,11 +95,11 @@ func init() { type saveCp struct { tableName string - merger TableCheckpointMerger + merger checkpoints.TableCheckpointMerger } type errorSummary struct { - status CheckpointStatus + status checkpoints.CheckpointStatus err error } @@ -125,7 +134,7 @@ func (es *errorSummaries) emitLog() { } } -func (es *errorSummaries) record(tableName string, err error, status CheckpointStatus) { +func (es *errorSummaries) record(tableName string, err error, status checkpoints.CheckpointStatus) { es.Lock() defer es.Unlock() es.summary[tableName] = errorSummary{status: status, err: err} @@ -137,6 +146,7 @@ const ( diskQuotaStateImporting ) +<<<<<<< HEAD type RestoreController struct { cfg *config.Config dbMetas []*mydump.MDDatabaseMeta @@ -154,19 +164,37 @@ type RestoreController struct { compactState int32 sysVars map[string]string tls *common.TLS +======= +type Controller struct { + cfg *config.Config + dbMetas []*mydump.MDDatabaseMeta + dbInfos map[string]*checkpoints.TidbDBInfo + tableWorkers *worker.Pool + indexWorkers *worker.Pool + regionWorkers *worker.Pool + ioWorkers *worker.Pool + checksumWorks *worker.Pool + pauser *common.Pauser + backend backend.Backend + tidbGlue glue.Glue + + alterTableLock sync.Mutex + sysVars map[string]string + tls *common.TLS +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) errorSummaries errorSummaries - checkpointsDB CheckpointsDB + checkpointsDB checkpoints.DB saveCpCh chan saveCp checkpointsWg sync.WaitGroup closedEngineLimit *worker.Pool store storage.ExternalStorage - checksumManager ChecksumManager diskQuotaLock sync.RWMutex diskQuotaState int32 + compactState int32 // commit ts for local and importer backend ts uint64 @@ -178,7 +206,7 @@ func NewRestoreController( cfg *config.Config, s storage.ExternalStorage, g glue.Glue, -) (*RestoreController, error) { +) (*Controller, error) { return NewRestoreControllerWithPauser(ctx, dbMetas, cfg, s, DeliverPauser, g) } @@ -189,7 +217,7 @@ func NewRestoreControllerWithPauser( s storage.ExternalStorage, pauser *common.Pauser, g glue.Glue, -) (*RestoreController, error) { +) (*Controller, error) { tls, err := cfg.ToTLS() if err != nil { return nil, err @@ -262,7 +290,7 @@ func NewRestoreControllerWithPauser( ts = oracle.ComposeTS(physical, logical) } - rc := &RestoreController{ + rc := &Controller{ cfg: cfg, dbMetas: dbMetas, tableWorkers: worker.NewPool(ctx, cfg.App.TableConcurrency, "table"), @@ -288,12 +316,12 @@ func NewRestoreControllerWithPauser( return rc, nil } -func (rc *RestoreController) Close() { +func (rc *Controller) Close() { rc.backend.Close() rc.tidbGlue.GetSQLExecutor().Close() } -func (rc *RestoreController) Run(ctx context.Context) error { +func (rc *Controller) Run(ctx context.Context) error { opts := []func(context.Context) error{ rc.checkRequirements, rc.setGlobalVariables, @@ -473,7 +501,7 @@ func (worker *restoreSchemaWorker) makeJobs(dbMetas []*mydump.MDDatabaseMeta) er } func (worker *restoreSchemaWorker) doJob() { - var session Session + var session checkpoints.Session defer func() { if session != nil { session.Close() @@ -571,7 +599,31 @@ func (worker *restoreSchemaWorker) appendJob(job *schemaJob) error { } } +<<<<<<< HEAD func (rc *RestoreController) restoreSchema(ctx context.Context) error { +======= +func (rc *Controller) checkTableEmpty(ctx context.Context, tableName string) error { + db, err := rc.tidbGlue.GetDB() + if err != nil { + return err + } + + query := "select 1 from " + tableName + " limit 1" + var dump int + err = db.QueryRowContext(ctx, query).Scan(&dump) + + switch { + case err == sql.ErrNoRows: + return nil + case err != nil: + return errors.AddStack(err) + default: + return errors.Errorf("table %s not empty, please clean up the table first", tableName) + } +} + +func (rc *Controller) restoreSchema(ctx context.Context) error { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) if !rc.cfg.Mydumper.NoSchema { logTask := log.L().Begin(zap.InfoLevel, "restore all schema") concurrency := utils.MinInt(rc.cfg.App.RegionConcurrency, 8) @@ -613,6 +665,34 @@ func (rc *RestoreController) restoreSchema(ctx context.Context) error { os.Exit(0) }) +<<<<<<< HEAD +======= + if rc.cfg.TikvImporter.Backend != config.BackendTiDB { + for _, dbMeta := range rc.dbMetas { + for _, tableMeta := range dbMeta.Tables { + tableName := common.UniqueTable(dbMeta.Name, tableMeta.Name) + + // if checkpoint enable and not missing, we skip the check table empty progress. + if rc.cfg.Checkpoint.Enable { + dbCp, err := rc.checkpointsDB.Get(ctx, tableName) + if err != nil { + return errors.Trace(err) + } + + if dbCp.Status > checkpoints.CheckpointStatusMissing { + continue + } + } + + err := rc.checkTableEmpty(ctx, tableName) + if err != nil { + return err + } + } + } + } + +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) go rc.listenCheckpointUpdates() rc.sysVars = ObtainImportantVariables(ctx, rc.tidbGlue.GetSQLExecutor()) @@ -623,7 +703,7 @@ func (rc *RestoreController) restoreSchema(ctx context.Context) error { } // verifyCheckpoint check whether previous task checkpoint is compatible with task config -func verifyCheckpoint(cfg *config.Config, taskCp *TaskCheckpoint) error { +func verifyCheckpoint(cfg *config.Config, taskCp *checkpoints.TaskCheckpoint) error { if taskCp == nil { return nil } @@ -679,15 +759,20 @@ func verifyCheckpoint(cfg *config.Config, taskCp *TaskCheckpoint) error { } // for local backend, we should check if local SST exists in disk, otherwise we'll lost data -func verifyLocalFile(ctx context.Context, cpdb CheckpointsDB, dir string) error { +func verifyLocalFile(ctx context.Context, cpdb checkpoints.DB, dir string) error { targetTables, err := cpdb.GetLocalStoringTables(ctx) if err != nil { return errors.Trace(err) } for tableName, engineIDs := range targetTables { for _, engineID := range engineIDs { +<<<<<<< HEAD _, eID := kv.MakeUUID(tableName, engineID) file := kv.LocalFile{Uuid: eID} +======= + _, eID := backend.MakeUUID(tableName, engineID) + file := local.File{UUID: eID} +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) err := file.Exist(dir) if err != nil { log.L().Error("can't find local file", @@ -700,7 +785,7 @@ func verifyLocalFile(ctx context.Context, cpdb CheckpointsDB, dir string) error return nil } -func (rc *RestoreController) estimateChunkCountIntoMetrics(ctx context.Context) error { +func (rc *Controller) estimateChunkCountIntoMetrics(ctx context.Context) error { estimatedChunkCount := 0.0 estimatedEngineCnt := int64(0) batchSize := int64(rc.cfg.Mydumper.BatchSize) @@ -713,11 +798,11 @@ func (rc *RestoreController) estimateChunkCountIntoMetrics(ctx context.Context) } fileChunks := make(map[string]float64) - for engineId, eCp := range dbCp.Engines { - if eCp.Status < CheckpointStatusImported { + for engineID, eCp := range dbCp.Engines { + if eCp.Status < checkpoints.CheckpointStatusImported { estimatedEngineCnt++ } - if engineId == indexEngineID { + if engineID == indexEngineID { continue } for _, c := range eCp.Chunks { @@ -742,10 +827,10 @@ func (rc *RestoreController) estimateChunkCountIntoMetrics(ctx context.Context) if fileMeta.FileMeta.FileSize > int64(cfg.MaxRegionSize) && cfg.StrictFormat && !cfg.CSV.Header { estimatedChunkCount += math.Round(float64(fileMeta.FileMeta.FileSize) / float64(cfg.MaxRegionSize)) } else { - estimatedChunkCount += 1 + estimatedChunkCount++ } } else { - estimatedChunkCount += 1 + estimatedChunkCount++ } } } @@ -757,8 +842,8 @@ func (rc *RestoreController) estimateChunkCountIntoMetrics(ctx context.Context) return nil } -func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int32, err error, statusIfSucceed CheckpointStatus) { - merger := &StatusCheckpointMerger{Status: statusIfSucceed, EngineID: engineID} +func (rc *Controller) saveStatusCheckpoint(tableName string, engineID int32, err error, statusIfSucceed checkpoints.CheckpointStatus) { + merger := &checkpoints.StatusCheckpointMerger{Status: statusIfSucceed, EngineID: engineID} log.L().Debug("update checkpoint", zap.String("table", tableName), zap.Int32("engine_id", engineID), zap.Uint8("new_status", uint8(statusIfSucceed)), zap.Error(err)) @@ -773,7 +858,7 @@ func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int return } - if engineID == WholeTableEngineID { + if engineID == checkpoints.WholeTableEngineID { metric.RecordTableCount(statusIfSucceed.MetricName(), err) } else { metric.RecordEngineCount(statusIfSucceed.MetricName(), err) @@ -783,11 +868,11 @@ func (rc *RestoreController) saveStatusCheckpoint(tableName string, engineID int } // listenCheckpointUpdates will combine several checkpoints together to reduce database load. -func (rc *RestoreController) listenCheckpointUpdates() { +func (rc *Controller) listenCheckpointUpdates() { rc.checkpointsWg.Add(1) var lock sync.Mutex - coalesed := make(map[string]*TableCheckpointDiff) + coalesed := make(map[string]*checkpoints.TableCheckpointDiff) hasCheckpoint := make(chan struct{}, 1) defer close(hasCheckpoint) @@ -796,7 +881,7 @@ func (rc *RestoreController) listenCheckpointUpdates() { for range hasCheckpoint { lock.Lock() cpd := coalesed - coalesed = make(map[string]*TableCheckpointDiff) + coalesed = make(map[string]*checkpoints.TableCheckpointDiff) lock.Unlock() if len(cpd) > 0 { @@ -811,7 +896,7 @@ func (rc *RestoreController) listenCheckpointUpdates() { lock.Lock() cpd, ok := coalesed[scp.tableName] if !ok { - cpd = NewTableCheckpointDiff() + cpd = checkpoints.NewTableCheckpointDiff() coalesed[scp.tableName] = cpd } scp.merger.MergeInto(cpd) @@ -823,42 +908,48 @@ func (rc *RestoreController) listenCheckpointUpdates() { lock.Unlock() + //nolint:scopelint // This would be either INLINED or ERASED, at compile time. failpoint.Inject("FailIfImportedChunk", func(val failpoint.Value) { - if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(val.(int)) { + if merger, ok := scp.merger.(*checkpoints.ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(val.(int)) { rc.checkpointsWg.Done() rc.checkpointsWg.Wait() panic("forcing failure due to FailIfImportedChunk") } }) + //nolint:scopelint // This would be either INLINED or ERASED, at compile time. failpoint.Inject("FailIfStatusBecomes", func(val failpoint.Value) { - if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && merger.EngineID >= 0 && int(merger.Status) == val.(int) { + if merger, ok := scp.merger.(*checkpoints.StatusCheckpointMerger); ok && merger.EngineID >= 0 && int(merger.Status) == val.(int) { rc.checkpointsWg.Done() rc.checkpointsWg.Wait() panic("forcing failure due to FailIfStatusBecomes") } }) + //nolint:scopelint // This would be either INLINED or ERASED, at compile time. failpoint.Inject("FailIfIndexEngineImported", func(val failpoint.Value) { - if merger, ok := scp.merger.(*StatusCheckpointMerger); ok && - merger.EngineID == WholeTableEngineID && - merger.Status == CheckpointStatusIndexImported && val.(int) > 0 { + if merger, ok := scp.merger.(*checkpoints.StatusCheckpointMerger); ok && + merger.EngineID == checkpoints.WholeTableEngineID && + merger.Status == checkpoints.CheckpointStatusIndexImported && val.(int) > 0 { rc.checkpointsWg.Done() rc.checkpointsWg.Wait() panic("forcing failure due to FailIfIndexEngineImported") } }) + //nolint:scopelint // This would be either INLINED or ERASED, at compile time. failpoint.Inject("KillIfImportedChunk", func(val failpoint.Value) { - if merger, ok := scp.merger.(*ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(val.(int)) { - common.KillMySelf() + if merger, ok := scp.merger.(*checkpoints.ChunkCheckpointMerger); ok && merger.Checksum.SumKVS() >= uint64(val.(int)) { + if err := common.KillMySelf(); err != nil { + log.L().Warn("KillMySelf() failed to kill itself", log.ShortError(err)) + } } }) } rc.checkpointsWg.Done() } -func (rc *RestoreController) runPeriodicActions(ctx context.Context, stop <-chan struct{}) { +func (rc *Controller) runPeriodicActions(ctx context.Context, stop <-chan struct{}) { // a nil channel blocks forever. // if the cron duration is zero we use the nil channel to skip the action. var logProgressChan <-chan time.Time @@ -928,16 +1019,16 @@ func (rc *RestoreController) runPeriodicActions(ctx context.Context, stop <-chan var state string var remaining zap.Field - if finished >= estimated { + switch { + case finished >= estimated: if engineFinished < engineEstimated { state = "importing" } else { state = "post-processing" } - remaining = zap.Skip() - } else if finished > 0 { + case finished > 0: state = "writing" - } else { + default: state = "preparing" } @@ -1000,7 +1091,7 @@ func (rc *RestoreController) runPeriodicActions(ctx context.Context, stop <-chan var checksumManagerKey struct{} -func (rc *RestoreController) restoreTables(ctx context.Context) error { +func (rc *Controller) restoreTables(ctx context.Context) error { logTask := log.L().Begin(zap.InfoLevel, "restore all tables data") // for local backend, we should disable some pd scheduler and change some settings, to @@ -1031,7 +1122,7 @@ func (rc *RestoreController) restoreTables(ctx context.Context) error { type task struct { tr *TableRestore - cp *TableCheckpoint + cp *checkpoints.TableCheckpoint } totalTables := 0 @@ -1075,7 +1166,7 @@ func (rc *RestoreController) restoreTables(ctx context.Context) error { } // first collect all tables where the checkpoint is invalid - allInvalidCheckpoints := make(map[string]CheckpointStatus) + allInvalidCheckpoints := make(map[string]checkpoints.CheckpointStatus) // collect all tables whose checkpoint's tableID can't match current tableID allDirtyCheckpoints := make(map[string]struct{}) for _, dbMeta := range rc.dbMetas { @@ -1094,7 +1185,7 @@ func (rc *RestoreController) restoreTables(ctx context.Context) error { if err != nil { return errors.Trace(err) } - if cp.Status <= CheckpointStatusMaxInvalid { + if cp.Status <= checkpoints.CheckpointStatusMaxInvalid { allInvalidCheckpoints[tableName] = cp.Status } else if cp.TableID > 0 && cp.TableID != tableInfo.ID { allDirtyCheckpoints[tableName] = struct{}{} @@ -1114,7 +1205,7 @@ func (rc *RestoreController) restoreTables(ctx context.Context) error { var action strings.Builder action.WriteString("./tidb-lightning-ctl --checkpoint-error-") switch failedStep { - case CheckpointStatusAlteredAutoInc, CheckpointStatusAnalyzed: + case checkpoints.CheckpointStatusAlteredAutoInc, checkpoints.CheckpointStatusAnalyzed: action.WriteString("ignore") default: action.WriteString("destroy") @@ -1211,10 +1302,10 @@ func (rc *RestoreController) restoreTables(ctx context.Context) error { return err } -func (t *TableRestore) restoreTable( +func (tr *TableRestore) restoreTable( ctx context.Context, - rc *RestoreController, - cp *TableCheckpoint, + rc *Controller, + cp *checkpoints.TableCheckpoint, ) (bool, error) { // 1. Load the table info. @@ -1226,49 +1317,53 @@ func (t *TableRestore) restoreTable( // no need to do anything if the chunks are already populated if len(cp.Engines) > 0 { - t.logger.Info("reusing engines and files info from checkpoint", + tr.logger.Info("reusing engines and files info from checkpoint", zap.Int("enginesCnt", len(cp.Engines)), zap.Int("filesCnt", cp.CountChunks()), ) - } else if cp.Status < CheckpointStatusAllWritten { - if err := t.populateChunks(ctx, rc, cp); err != nil { + } else if cp.Status < checkpoints.CheckpointStatusAllWritten { + if err := tr.populateChunks(ctx, rc, cp); err != nil { return false, errors.Trace(err) } - if err := rc.checkpointsDB.InsertEngineCheckpoints(ctx, t.tableName, cp.Engines); err != nil { + if err := rc.checkpointsDB.InsertEngineCheckpoints(ctx, tr.tableName, cp.Engines); err != nil { return false, errors.Trace(err) } - web.BroadcastTableCheckpoint(t.tableName, cp) + web.BroadcastTableCheckpoint(tr.tableName, cp) // rebase the allocator so it exceeds the number of rows. - if t.tableInfo.Core.PKIsHandle && t.tableInfo.Core.ContainsAutoRandomBits() { - cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, t.tableInfo.Core.AutoRandID) - t.alloc.Get(autoid.AutoRandomType).Rebase(t.tableInfo.ID, cp.AllocBase, false) + if tr.tableInfo.Core.PKIsHandle && tr.tableInfo.Core.ContainsAutoRandomBits() { + cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, tr.tableInfo.Core.AutoRandID) + if err := tr.alloc.Get(autoid.AutoRandomType).Rebase(tr.tableInfo.ID, cp.AllocBase, false); err != nil { + return false, err + } } else { - cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, t.tableInfo.Core.AutoIncID) - t.alloc.Get(autoid.RowIDAllocType).Rebase(t.tableInfo.ID, cp.AllocBase, false) + cp.AllocBase = mathutil.MaxInt64(cp.AllocBase, tr.tableInfo.Core.AutoIncID) + if err := tr.alloc.Get(autoid.RowIDAllocType).Rebase(tr.tableInfo.ID, cp.AllocBase, false); err != nil { + return false, err + } } rc.saveCpCh <- saveCp{ - tableName: t.tableName, - merger: &RebaseCheckpointMerger{ + tableName: tr.tableName, + merger: &checkpoints.RebaseCheckpointMerger{ AllocBase: cp.AllocBase, }, } } // 2. Restore engines (if still needed) - err := t.restoreEngines(ctx, rc, cp) + err := tr.restoreEngines(ctx, rc, cp) if err != nil { return false, errors.Trace(err) } // 3. Post-process. With the last parameter set to false, we can allow delay analyze execute latter - return t.postProcess(ctx, rc, cp, false /* force-analyze */) + return tr.postProcess(ctx, rc, cp, false /* force-analyze */) } -func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreController, cp *TableCheckpoint) error { +func (tr *TableRestore) restoreEngines(pCtx context.Context, rc *Controller, cp *checkpoints.TableCheckpoint) error { indexEngineCp := cp.Engines[indexEngineID] if indexEngineCp == nil { - return errors.Errorf("table %v index engine checkpoint not found", t.tableName) + return errors.Errorf("table %v index engine checkpoint not found", tr.tableName) } ctx, cancel := context.WithCancel(pCtx) @@ -1286,6 +1381,7 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle // if index-engine checkpoint is lower than `CheckpointStatusClosed`, there must be // data-engines that need to be restore or import. Otherwise, all data-engines should // be finished already. +<<<<<<< HEAD if indexEngineCp.Status < CheckpointStatusClosed { indexWorker := rc.indexWorkers.Apply() defer rc.indexWorkers.Recycle(indexWorker) @@ -1301,9 +1397,31 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle if indexEngine == nil { return errors.Errorf("table checkpoint status %v incompitable with index engine checkpoint status %v", cp.Status, indexEngineCp.Status) +======= + + if indexEngineCp.Status < checkpoints.CheckpointStatusClosed { + indexWorker := rc.indexWorkers.Apply() + defer rc.indexWorkers.Recycle(indexWorker) + + // import backend can't reopen engine if engine is closed, so + // only open index engine if any data engines don't finish writing. + var indexEngine *backend.OpenedEngine + var err error + for engineID, engine := range cp.Engines { + if engineID == indexEngineID { + continue + } + if engine.Status < checkpoints.CheckpointStatusAllWritten { + indexEngine, err = rc.backend.OpenEngine(ctx, tr.tableName, indexEngineID, rc.ts) + if err != nil { + return errors.Trace(err) + } + break + } +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) } - logTask := t.logger.Begin(zap.InfoLevel, "import whole table") + logTask := tr.logger.Begin(zap.InfoLevel, "import whole table") var wg sync.WaitGroup var engineErr common.OnceError setError := func(err error) { @@ -1314,7 +1432,7 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle type engineCheckpoint struct { engineID int32 - checkpoint *EngineCheckpoint + checkpoint *checkpoints.EngineCheckpoint } allEngines := make([]engineCheckpoint, 0, len(cp.Engines)) for engineID, engine := range cp.Engines { @@ -1342,7 +1460,7 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle continue } - if engine.Status < CheckpointStatusImported { + if engine.Status < checkpoints.CheckpointStatusImported { wg.Add(1) // Note: We still need tableWorkers to control the concurrency of tables. @@ -1350,11 +1468,11 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle // the difference between restoring tables concurrently and restoring tables one by one. restoreWorker := rc.tableWorkers.Apply() - go func(w *worker.Worker, eid int32, ecp *EngineCheckpoint) { + go func(w *worker.Worker, eid int32, ecp *checkpoints.EngineCheckpoint) { defer wg.Done() - engineLogTask := t.logger.With(zap.Int32("engineNumber", eid)).Begin(zap.InfoLevel, "restore engine") - dataClosedEngine, err := t.restoreEngine(ctx, rc, indexEngine, eid, ecp) + engineLogTask := tr.logger.With(zap.Int32("engineNumber", eid)).Begin(zap.InfoLevel, "restore engine") + dataClosedEngine, err := tr.restoreEngine(ctx, rc, indexEngine, eid, ecp) engineLogTask.End(zap.ErrorLevel, err) rc.tableWorkers.Recycle(w) if err != nil { @@ -1368,7 +1486,7 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle dataWorker := rc.closedEngineLimit.Apply() defer rc.closedEngineLimit.Recycle(dataWorker) - if err := t.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { + if err := tr.importEngine(ctx, dataClosedEngine, rc, eid, ecp); err != nil { setError(err) } }(restoreWorker, engineID, engine) @@ -1383,19 +1501,31 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle return errors.Trace(restoreErr) } +<<<<<<< HEAD closedIndexEngine, restoreErr = indexEngine.Close(ctx) rc.saveStatusCheckpoint(t.tableName, indexEngineID, err, CheckpointStatusClosed) } else if indexEngineCp.Status == CheckpointStatusClosed { +======= + if indexEngine != nil { + closedIndexEngine, restoreErr = indexEngine.Close(ctx) + } else { + closedIndexEngine, restoreErr = rc.backend.UnsafeCloseEngine(ctx, tr.tableName, indexEngineID) + } + + rc.saveStatusCheckpoint(tr.tableName, indexEngineID, restoreErr, checkpoints.CheckpointStatusClosed) + } else if indexEngineCp.Status == checkpoints.CheckpointStatusClosed { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) // If index engine file has been closed but not imported only if context cancel occurred // when `importKV()` execution, so `UnsafeCloseEngine` and continue import it. - closedIndexEngine, restoreErr = rc.backend.UnsafeCloseEngine(ctx, t.tableName, indexEngineID) + closedIndexEngine, restoreErr = rc.backend.UnsafeCloseEngine(ctx, tr.tableName, indexEngineID) } if restoreErr != nil { return errors.Trace(restoreErr) } - if cp.Status < CheckpointStatusIndexImported { + if cp.Status < checkpoints.CheckpointStatusIndexImported { var err error +<<<<<<< HEAD if indexEngineCp.Status < CheckpointStatusImported { // the lock ensures the import() step will not be concurrent. if !rc.isLocalBackend() { @@ -1406,13 +1536,17 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle if !rc.isLocalBackend() { rc.postProcessLock.Unlock() } +======= + if indexEngineCp.Status < checkpoints.CheckpointStatusImported { + err = tr.importKV(ctx, closedIndexEngine, rc, indexEngineID) +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) } failpoint.Inject("FailBeforeIndexEngineImported", func() { panic("forcing failure due to FailBeforeIndexEngineImported") }) - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, err, CheckpointStatusIndexImported) + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusIndexImported) if err != nil { return errors.Trace(err) } @@ -1420,18 +1554,26 @@ func (t *TableRestore) restoreEngines(pCtx context.Context, rc *RestoreControlle return nil } -func (t *TableRestore) restoreEngine( +func (tr *TableRestore) restoreEngine( pCtx context.Context, +<<<<<<< HEAD rc *RestoreController, indexEngine *kv.OpenedEngine, engineID int32, cp *EngineCheckpoint, ) (*kv.ClosedEngine, error) { +======= + rc *Controller, + indexEngine *backend.OpenedEngine, + engineID int32, + cp *checkpoints.EngineCheckpoint, +) (*backend.ClosedEngine, error) { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) ctx, cancel := context.WithCancel(pCtx) defer cancel() // all data has finished written, we can close the engine directly. - if cp.Status >= CheckpointStatusAllWritten { - closedEngine, err := rc.backend.UnsafeCloseEngine(ctx, t.tableName, engineID) + if cp.Status >= checkpoints.CheckpointStatusAllWritten { + closedEngine, err := rc.backend.UnsafeCloseEngine(ctx, tr.tableName, engineID) // If any error occurred, recycle worker immediately if err != nil { return closedEngine, errors.Trace(err) @@ -1439,9 +1581,9 @@ func (t *TableRestore) restoreEngine( return closedEngine, nil } - logTask := t.logger.With(zap.Int32("engineNumber", engineID)).Begin(zap.InfoLevel, "encode kv data and write") + logTask := tr.logger.With(zap.Int32("engineNumber", engineID)).Begin(zap.InfoLevel, "encode kv data and write") - dataEngine, err := rc.backend.OpenEngine(ctx, t.tableName, engineID, rc.ts) + dataEngine, err := rc.backend.OpenEngine(ctx, tr.tableName, engineID, rc.ts) if err != nil { return nil, errors.Trace(err) } @@ -1470,7 +1612,7 @@ func (t *TableRestore) restoreEngine( // 2. sql -> kvs // 3. load kvs data (into kv deliver server) // 4. flush kvs data (into tikv node) - cr, err := newChunkRestore(ctx, chunkIndex, rc.cfg, chunk, rc.ioWorkers, rc.store, t.tableInfo) + cr, err := newChunkRestore(ctx, chunkIndex, rc.cfg, chunk, rc.ioWorkers, rc.store, tr.tableInfo) if err != nil { return nil, errors.Trace(err) } @@ -1498,7 +1640,7 @@ func (t *TableRestore) restoreEngine( rc.regionWorkers.Recycle(w) }() metric.ChunkCounter.WithLabelValues(metric.ChunkStateRunning).Add(remainChunkCnt) - err := cr.restore(ctx, t, engineID, dataWriter, indexWriter, rc) + err := cr.restore(ctx, tr, engineID, dataWriter, indexWriter, rc) if err == nil { err = dataWriter.Close() } @@ -1538,7 +1680,7 @@ func (t *TableRestore) restoreEngine( } // Currently we write all the checkpoints after data&index engine are flushed. for _, chunk := range cp.Chunks { - saveCheckpoint(rc, t, engineID, chunk) + saveCheckpoint(rc, tr, engineID, chunk) } return nil } @@ -1547,7 +1689,7 @@ func (t *TableRestore) restoreEngine( // so there may be data lose if exit at here. So we don't write this checkpoint // here like other mode. if !rc.isLocalBackend() { - rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusAllWritten) + rc.saveStatusCheckpoint(tr.tableName, engineID, err, checkpoints.CheckpointStatusAllWritten) } if err != nil { // if process is canceled, we should flush all chunk checkpoints for local backend @@ -1574,10 +1716,10 @@ func (t *TableRestore) restoreEngine( // Currently we write all the checkpoints after data&index engine are flushed. for _, chunk := range cp.Chunks { - saveCheckpoint(rc, t, engineID, chunk) + saveCheckpoint(rc, tr, engineID, chunk) } } - rc.saveStatusCheckpoint(t.tableName, engineID, err, CheckpointStatusClosed) + rc.saveStatusCheckpoint(tr.tableName, engineID, err, checkpoints.CheckpointStatusClosed) if err != nil { // If any error occurred, recycle worker immediately return nil, errors.Trace(err) @@ -1585,17 +1727,23 @@ func (t *TableRestore) restoreEngine( return closedDataEngine, nil } -func (t *TableRestore) importEngine( +func (tr *TableRestore) importEngine( ctx context.Context, +<<<<<<< HEAD closedEngine *kv.ClosedEngine, rc *RestoreController, +======= + closedEngine *backend.ClosedEngine, + rc *Controller, +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) engineID int32, - cp *EngineCheckpoint, + cp *checkpoints.EngineCheckpoint, ) error { - if cp.Status >= CheckpointStatusImported { + if cp.Status >= checkpoints.CheckpointStatusImported { return nil } +<<<<<<< HEAD // 1. close engine, then calling import // FIXME: flush is an asynchronous operation, what if flush failed? @@ -1609,6 +1757,10 @@ func (t *TableRestore) importEngine( rc.postProcessLock.Unlock() } if err != nil { +======= + // 1. calling import + if err := tr.importKV(ctx, closedEngine, rc, engineID); err != nil { +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) return errors.Trace(err) } @@ -1630,10 +1782,10 @@ func (t *TableRestore) importEngine( // // if the parameter forcePostProcess to true, postProcess force run checksum and analyze even if the // post-process-at-last config is true. And if this two phases are skipped, the first return value will be true. -func (t *TableRestore) postProcess( +func (tr *TableRestore) postProcess( ctx context.Context, - rc *RestoreController, - cp *TableCheckpoint, + rc *Controller, + cp *checkpoints.TableCheckpoint, forcePostProcess bool, ) (bool, error) { // there are no data in this table, no need to do post process @@ -1644,28 +1796,28 @@ func (t *TableRestore) postProcess( } // 3. alter table set auto_increment - if cp.Status < CheckpointStatusAlteredAutoInc { + if cp.Status < checkpoints.CheckpointStatusAlteredAutoInc { rc.alterTableLock.Lock() - tblInfo := t.tableInfo.Core + tblInfo := tr.tableInfo.Core var err error if tblInfo.PKIsHandle && tblInfo.ContainsAutoRandomBits() { - err = AlterAutoRandom(ctx, rc.tidbGlue.GetSQLExecutor(), t.tableName, t.alloc.Get(autoid.AutoRandomType).Base()+1) + err = AlterAutoRandom(ctx, rc.tidbGlue.GetSQLExecutor(), tr.tableName, tr.alloc.Get(autoid.AutoRandomType).Base()+1) } else if common.TableHasAutoRowID(tblInfo) || tblInfo.GetAutoIncrementColInfo() != nil { // only alter auto increment id iff table contains auto-increment column or generated handle - err = AlterAutoIncrement(ctx, rc.tidbGlue.GetSQLExecutor(), t.tableName, t.alloc.Get(autoid.RowIDAllocType).Base()+1) + err = AlterAutoIncrement(ctx, rc.tidbGlue.GetSQLExecutor(), tr.tableName, tr.alloc.Get(autoid.RowIDAllocType).Base()+1) } rc.alterTableLock.Unlock() - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, err, CheckpointStatusAlteredAutoInc) + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusAlteredAutoInc) if err != nil { return false, err } - cp.Status = CheckpointStatusAlteredAutoInc + cp.Status = checkpoints.CheckpointStatusAlteredAutoInc } // tidb backend don't need checksum & analyze if !rc.backend.ShouldPostProcess() { - t.logger.Debug("skip checksum & analyze, not supported by this backend") - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, nil, CheckpointStatusAnalyzeSkipped) + tr.logger.Debug("skip checksum & analyze, not supported by this backend") + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusAnalyzeSkipped) return false, nil } @@ -1673,10 +1825,10 @@ func (t *TableRestore) postProcess( defer rc.checksumWorks.Recycle(w) finished := true - if cp.Status < CheckpointStatusChecksummed { + if cp.Status < checkpoints.CheckpointStatusChecksummed { if rc.cfg.PostRestore.Checksum == config.OpLevelOff { - t.logger.Info("skip checksum") - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, nil, CheckpointStatusChecksumSkipped) + tr.logger.Info("skip checksum") + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusChecksumSkipped) } else { if forcePostProcess || !rc.cfg.PostRestore.PostProcessAtLast { // 4. do table checksum @@ -1686,20 +1838,20 @@ func (t *TableRestore) postProcess( localChecksum.Add(&chunk.Checksum) } } - t.logger.Info("local checksum", zap.Object("checksum", &localChecksum)) - err := t.compareChecksum(ctx, localChecksum) + tr.logger.Info("local checksum", zap.Object("checksum", &localChecksum)) + err := tr.compareChecksum(ctx, localChecksum) // with post restore level 'optional', we will skip checksum error if rc.cfg.PostRestore.Checksum == config.OpLevelOptional { if err != nil { - t.logger.Warn("compare checksum failed, will skip this error and go on", log.ShortError(err)) + tr.logger.Warn("compare checksum failed, will skip this error and go on", log.ShortError(err)) err = nil } } - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, err, CheckpointStatusChecksummed) + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusChecksummed) if err != nil { return false, errors.Trace(err) } - cp.Status = CheckpointStatusChecksummed + cp.Status = checkpoints.CheckpointStatusChecksummed } else { finished = false } @@ -1710,26 +1862,27 @@ func (t *TableRestore) postProcess( } // 5. do table analyze - if cp.Status < CheckpointStatusAnalyzed { - if rc.cfg.PostRestore.Analyze == config.OpLevelOff { - t.logger.Info("skip analyze") - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, nil, CheckpointStatusAnalyzeSkipped) - cp.Status = CheckpointStatusAnalyzed - } else if forcePostProcess || !rc.cfg.PostRestore.PostProcessAtLast { - err := t.analyzeTable(ctx, rc.tidbGlue.GetSQLExecutor()) + if cp.Status < checkpoints.CheckpointStatusAnalyzed { + switch { + case rc.cfg.PostRestore.Analyze == config.OpLevelOff: + tr.logger.Info("skip analyze") + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, nil, checkpoints.CheckpointStatusAnalyzeSkipped) + cp.Status = checkpoints.CheckpointStatusAnalyzed + case forcePostProcess || !rc.cfg.PostRestore.PostProcessAtLast: + err := tr.analyzeTable(ctx, rc.tidbGlue.GetSQLExecutor()) // witch post restore level 'optional', we will skip analyze error if rc.cfg.PostRestore.Analyze == config.OpLevelOptional { if err != nil { - t.logger.Warn("analyze table failed, will skip this error and go on", log.ShortError(err)) + tr.logger.Warn("analyze table failed, will skip this error and go on", log.ShortError(err)) err = nil } } - rc.saveStatusCheckpoint(t.tableName, WholeTableEngineID, err, CheckpointStatusAnalyzed) + rc.saveStatusCheckpoint(tr.tableName, checkpoints.WholeTableEngineID, err, checkpoints.CheckpointStatusAnalyzed) if err != nil { return false, errors.Trace(err) } - cp.Status = CheckpointStatusAnalyzed - } else { + cp.Status = checkpoints.CheckpointStatusAnalyzed + default: finished = false } } @@ -1738,7 +1891,7 @@ func (t *TableRestore) postProcess( } // do full compaction for the whole data. -func (rc *RestoreController) fullCompact(ctx context.Context) error { +func (rc *Controller) fullCompact(ctx context.Context) error { if !rc.cfg.PostRestore.Compact { log.L().Info("skip full compaction") return nil @@ -1754,7 +1907,7 @@ func (rc *RestoreController) fullCompact(ctx context.Context) error { return errors.Trace(rc.doCompact(ctx, FullLevelCompact)) } -func (rc *RestoreController) doCompact(ctx context.Context, level int32) error { +func (rc *Controller) doCompact(ctx context.Context, level int32) error { tls := rc.tls.WithHost(rc.cfg.TiDB.PdAddr) return kv.ForAllStores( ctx, @@ -1766,16 +1919,16 @@ func (rc *RestoreController) doCompact(ctx context.Context, level int32) error { ) } -func (rc *RestoreController) switchToImportMode(ctx context.Context) { +func (rc *Controller) switchToImportMode(ctx context.Context) { rc.switchTiKVMode(ctx, sstpb.SwitchMode_Import) } -func (rc *RestoreController) switchToNormalMode(ctx context.Context) error { +func (rc *Controller) switchToNormalMode(ctx context.Context) error { rc.switchTiKVMode(ctx, sstpb.SwitchMode_Normal) return nil } -func (rc *RestoreController) switchTiKVMode(ctx context.Context, mode sstpb.SwitchMode) { +func (rc *Controller) switchTiKVMode(ctx context.Context, mode sstpb.SwitchMode) { // It is fine if we miss some stores which did not switch to Import mode, // since we're running it periodically, so we exclude disconnected stores. // But it is essential all stores be switched back to Normal mode to allow @@ -1799,7 +1952,7 @@ func (rc *RestoreController) switchTiKVMode(ctx context.Context, mode sstpb.Swit ) } -func (rc *RestoreController) enforceDiskQuota(ctx context.Context) { +func (rc *Controller) enforceDiskQuota(ctx context.Context) { if !atomic.CompareAndSwapInt32(&rc.diskQuotaState, diskQuotaStateIdle, diskQuotaStateChecking) { // do not run multiple the disk quota check / import simultaneously. // (we execute the lock check in background to avoid blocking the cron thread) @@ -1884,7 +2037,7 @@ func (rc *RestoreController) enforceDiskQuota(ctx context.Context) { }() } -func (rc *RestoreController) checkRequirements(ctx context.Context) error { +func (rc *Controller) checkRequirements(ctx context.Context) error { // skip requirement check if explicitly turned off if !rc.cfg.App.CheckRequirements { return nil @@ -1892,7 +2045,7 @@ func (rc *RestoreController) checkRequirements(ctx context.Context) error { return rc.backend.CheckRequirements(ctx) } -func (rc *RestoreController) setGlobalVariables(ctx context.Context) error { +func (rc *Controller) setGlobalVariables(ctx context.Context) error { // set new collation flag base on tidb config enabled := ObtainNewCollationEnabled(ctx, rc.tidbGlue.GetSQLExecutor()) // we should enable/disable new collation here since in server mode, tidb config @@ -1901,13 +2054,13 @@ func (rc *RestoreController) setGlobalVariables(ctx context.Context) error { return nil } -func (rc *RestoreController) waitCheckpointFinish() { +func (rc *Controller) waitCheckpointFinish() { // wait checkpoint process finish so that we can do cleanup safely close(rc.saveCpCh) rc.checkpointsWg.Wait() } -func (rc *RestoreController) cleanCheckpoints(ctx context.Context) error { +func (rc *Controller) cleanCheckpoints(ctx context.Context) error { rc.waitCheckpointFinish() if !rc.cfg.Checkpoint.Enable { @@ -1930,24 +2083,24 @@ func (rc *RestoreController) cleanCheckpoints(ctx context.Context) error { return errors.Annotate(err, "clean checkpoints") } -func (rc *RestoreController) isLocalBackend() bool { +func (rc *Controller) isLocalBackend() bool { return rc.cfg.TikvImporter.Backend == "local" } type chunkRestore struct { parser mydump.Parser index int - chunk *ChunkCheckpoint + chunk *checkpoints.ChunkCheckpoint } func newChunkRestore( ctx context.Context, index int, cfg *config.Config, - chunk *ChunkCheckpoint, + chunk *checkpoints.ChunkCheckpoint, ioWorkers *worker.Pool, store storage.ExternalStorage, - tableInfo *TidbTableInfo, + tableInfo *checkpoints.TidbTableInfo, ) (*chunkRestore, error) { blockBufSize := int64(cfg.Mydumper.ReadBlockSize) @@ -1999,8 +2152,8 @@ func (cr *chunkRestore) close() { type TableRestore struct { // The unique table name in the form "`db`.`tbl`". tableName string - dbInfo *TidbDBInfo - tableInfo *TidbTableInfo + dbInfo *checkpoints.TidbDBInfo + tableInfo *checkpoints.TidbTableInfo tableMeta *mydump.MDTableMeta encTable table.Table alloc autoid.Allocators @@ -2010,9 +2163,9 @@ type TableRestore struct { func NewTableRestore( tableName string, tableMeta *mydump.MDTableMeta, - dbInfo *TidbDBInfo, - tableInfo *TidbTableInfo, - cp *TableCheckpoint, + dbInfo *checkpoints.TidbDBInfo, + tableInfo *checkpoints.TidbTableInfo, + cp *checkpoints.TableCheckpoint, ) (*TableRestore, error) { idAlloc := kv.NewPanickingAllocators(cp.AllocBase) tbl, err := tables.TableFromMeta(idAlloc, tableInfo.Core) @@ -2036,9 +2189,9 @@ func (tr *TableRestore) Close() { tr.logger.Info("restore done") } -func (t *TableRestore) populateChunks(ctx context.Context, rc *RestoreController, cp *TableCheckpoint) error { - task := t.logger.Begin(zap.InfoLevel, "load engines and files") - chunks, err := mydump.MakeTableRegions(ctx, t.tableMeta, len(t.tableInfo.Core.Columns), rc.cfg, rc.ioWorkers, rc.store) +func (tr *TableRestore) populateChunks(ctx context.Context, rc *Controller, cp *checkpoints.TableCheckpoint) error { + task := tr.logger.Begin(zap.InfoLevel, "load engines and files") + chunks, err := mydump.MakeTableRegions(ctx, tr.tableMeta, len(tr.tableInfo.Core.Columns), rc.cfg, rc.ioWorkers, rc.store) if err == nil { timestamp := time.Now().Unix() failpoint.Inject("PopulateChunkTimestamp", func(v failpoint.Value) { @@ -2047,13 +2200,13 @@ func (t *TableRestore) populateChunks(ctx context.Context, rc *RestoreController for _, chunk := range chunks { engine, found := cp.Engines[chunk.EngineID] if !found { - engine = &EngineCheckpoint{ - Status: CheckpointStatusLoaded, + engine = &checkpoints.EngineCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, } cp.Engines[chunk.EngineID] = engine } - ccp := &ChunkCheckpoint{ - Key: ChunkCheckpointKey{ + ccp := &checkpoints.ChunkCheckpoint{ + Key: checkpoints.ChunkCheckpointKey{ Path: chunk.FileMeta.Path, Offset: chunk.Chunk.Offset, }, @@ -2063,7 +2216,7 @@ func (t *TableRestore) populateChunks(ctx context.Context, rc *RestoreController Timestamp: timestamp, } if len(chunk.Chunk.Columns) > 0 { - perms, err := t.parseColumnPermutations(chunk.Chunk.Columns) + perms, err := tr.parseColumnPermutations(chunk.Chunk.Columns) if err != nil { return errors.Trace(err) } @@ -2073,7 +2226,7 @@ func (t *TableRestore) populateChunks(ctx context.Context, rc *RestoreController } // Add index engine checkpoint - cp.Engines[indexEngineID] = &EngineCheckpoint{Status: CheckpointStatusLoaded} + cp.Engines[indexEngineID] = &checkpoints.EngineCheckpoint{Status: checkpoints.CheckpointStatusLoaded} } task.End(zap.ErrorLevel, err, zap.Int("enginesCnt", len(cp.Engines)), @@ -2095,14 +2248,14 @@ func (t *TableRestore) populateChunks(ctx context.Context, rc *RestoreController // The column permutation of (d, b, a) is set to be [2, 1, -1, 0]. // // The argument `columns` _must_ be in lower case. -func (t *TableRestore) initializeColumns(columns []string, ccp *ChunkCheckpoint) error { +func (tr *TableRestore) initializeColumns(columns []string, ccp *checkpoints.ChunkCheckpoint) error { var colPerm []int if len(columns) == 0 { - colPerm = make([]int, 0, len(t.tableInfo.Core.Columns)+1) - shouldIncludeRowID := common.TableHasAutoRowID(t.tableInfo.Core) + colPerm = make([]int, 0, len(tr.tableInfo.Core.Columns)+1) + shouldIncludeRowID := common.TableHasAutoRowID(tr.tableInfo.Core) // no provided columns, so use identity permutation. - for i := range t.tableInfo.Core.Columns { + for i := range tr.tableInfo.Core.Columns { colPerm = append(colPerm, i) } if shouldIncludeRowID { @@ -2110,7 +2263,7 @@ func (t *TableRestore) initializeColumns(columns []string, ccp *ChunkCheckpoint) } } else { var err error - colPerm, err = t.parseColumnPermutations(columns) + colPerm, err = tr.parseColumnPermutations(columns) if err != nil { return errors.Trace(err) } @@ -2120,8 +2273,8 @@ func (t *TableRestore) initializeColumns(columns []string, ccp *ChunkCheckpoint) return nil } -func (t *TableRestore) parseColumnPermutations(columns []string) ([]int, error) { - colPerm := make([]int, 0, len(t.tableInfo.Core.Columns)+1) +func (tr *TableRestore) parseColumnPermutations(columns []string) ([]int, error) { + colPerm := make([]int, 0, len(tr.tableInfo.Core.Columns)+1) columnMap := make(map[string]int) for i, column := range columns { @@ -2129,7 +2282,7 @@ func (t *TableRestore) parseColumnPermutations(columns []string) ([]int, error) } tableColumnMap := make(map[string]int) - for i, col := range t.tableInfo.Core.Columns { + for i, col := range tr.tableInfo.Core.Columns { tableColumnMap[col.Name.L] = i } @@ -2144,12 +2297,12 @@ func (t *TableRestore) parseColumnPermutations(columns []string) ([]int, error) return colPerm, errors.Errorf("unknown columns in header %s", unknownCols) } - for _, colInfo := range t.tableInfo.Core.Columns { + for _, colInfo := range tr.tableInfo.Core.Columns { if i, ok := columnMap[colInfo.Name.L]; ok { colPerm = append(colPerm, i) } else { if len(colInfo.GeneratedExprString) == 0 { - t.logger.Warn("column missing from data file, going to fill with default value", + tr.logger.Warn("column missing from data file, going to fill with default value", zap.String("colName", colInfo.Name.O), zap.Stringer("colType", &colInfo.FieldType), ) @@ -2159,7 +2312,7 @@ func (t *TableRestore) parseColumnPermutations(columns []string) ([]int, error) } if i, ok := columnMap[model.ExtraHandleName.L]; ok { colPerm = append(colPerm, i) - } else if common.TableHasAutoRowID(t.tableInfo.Core) { + } else if common.TableHasAutoRowID(tr.tableInfo.Core) { colPerm = append(colPerm, -1) } @@ -2183,7 +2336,7 @@ func getColumnNames(tableInfo *model.TableInfo, permutation []int) []string { for _, idx := range colIndexes { // skip columns with index -1 if idx >= 0 { - // original fiels contains _tidb_rowid field + // original fields contains _tidb_rowid field if idx == len(tableInfo.Columns) { names = append(names, model.ExtraHandleName.O) } else { @@ -2196,16 +2349,21 @@ func getColumnNames(tableInfo *model.TableInfo, permutation []int) []string { func (tr *TableRestore) importKV( ctx context.Context, +<<<<<<< HEAD closedEngine *kv.ClosedEngine, rc *RestoreController, +======= + closedEngine *backend.ClosedEngine, + rc *Controller, +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) engineID int32, ) error { task := closedEngine.Logger().Begin(zap.InfoLevel, "import and cleanup engine") err := closedEngine.Import(ctx) - rc.saveStatusCheckpoint(tr.tableName, engineID, err, CheckpointStatusImported) + rc.saveStatusCheckpoint(tr.tableName, engineID, err, checkpoints.CheckpointStatusImported) if err == nil { - closedEngine.Cleanup(ctx) + err = closedEngine.Cleanup(ctx) } dur := task.End(zap.ErrorLevel, err) @@ -2249,8 +2407,6 @@ func (tr *TableRestore) analyzeTable(ctx context.Context, g glue.SQLExecutor) er return err } -//////////////////////////////////////////////////////////////// - var ( maxKVQueueSize = 128 // Cache at most this number of rows before blocking the encode loop minDeliverBytes uint64 = 96 * units.KiB // 96 KB (data + index). batch at least this amount of bytes to reduce number of messages @@ -2268,13 +2424,19 @@ type deliverResult struct { err error } +//nolint:nakedret // TODO: refactor func (cr *chunkRestore) deliverLoop( ctx context.Context, kvsCh <-chan []deliveredKVs, t *TableRestore, engineID int32, +<<<<<<< HEAD dataEngine, indexEngine *kv.LocalEngineWriter, rc *RestoreController, +======= + dataEngine, indexEngine *backend.LocalEngineWriter, + rc *Controller, +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) ) (deliverTotalDur time.Duration, err error) { var channelClosed bool @@ -2368,7 +2530,7 @@ func (cr *chunkRestore) deliverLoop( deliverLogger.Warn("Slowed down write rows") }) failpoint.Inject("FailAfterWriteRows", nil) - // TODO: for local backend, we may save checkpoint more frequently, e.g. after writen + // TODO: for local backend, we may save checkpoint more frequently, e.g. after written // 10GB kv pairs to data engine, we can do a flush for both data & index engine, then we // can safely update current checkpoint. @@ -2383,7 +2545,7 @@ func (cr *chunkRestore) deliverLoop( return } -func saveCheckpoint(rc *RestoreController, t *TableRestore, engineID int32, chunk *ChunkCheckpoint) { +func saveCheckpoint(rc *Controller, t *TableRestore, engineID int32, chunk *checkpoints.ChunkCheckpoint) { // We need to update the AllocBase every time we've finished a file. // The AllocBase is determined by the maximum of the "handle" (_tidb_rowid // or integer primary key), which can only be obtained by reading all data. @@ -2396,13 +2558,13 @@ func saveCheckpoint(rc *RestoreController, t *TableRestore, engineID int32, chun } rc.saveCpCh <- saveCp{ tableName: t.tableName, - merger: &RebaseCheckpointMerger{ + merger: &checkpoints.RebaseCheckpointMerger{ AllocBase: base, }, } rc.saveCpCh <- saveCp{ tableName: t.tableName, - merger: &ChunkCheckpointMerger{ + merger: &checkpoints.ChunkCheckpointMerger{ EngineID: engineID, Key: chunk.Key, Checksum: chunk.Checksum, @@ -2413,6 +2575,7 @@ func saveCheckpoint(rc *RestoreController, t *TableRestore, engineID int32, chun } } +//nolint:nakedret // TODO: refactor func (cr *chunkRestore) encodeLoop( ctx context.Context, kvsCh chan<- []deliveredKVs, @@ -2420,7 +2583,7 @@ func (cr *chunkRestore) encodeLoop( logger log.Logger, kvEncoder kv.Encoder, deliverCompleteCh <-chan deliverResult, - rc *RestoreController, + rc *Controller, ) (readTotalDur time.Duration, encodeTotalDur time.Duration, err error) { send := func(kvs []deliveredKVs) error { select { @@ -2517,8 +2680,13 @@ func (cr *chunkRestore) restore( ctx context.Context, t *TableRestore, engineID int32, +<<<<<<< HEAD dataEngine, indexEngine *kv.LocalEngineWriter, rc *RestoreController, +======= + dataEngine, indexEngine *backend.LocalEngineWriter, + rc *Controller, +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) ) error { // Create the encoder. kvEncoder, err := rc.backend.NewEncoder(t.encTable, &kv.SessionOptions{ diff --git a/pkg/lightning/restore/restore_test.go b/pkg/lightning/restore/restore_test.go index 9766fd6fb..54751d210 100644 --- a/pkg/lightning/restore/restore_test.go +++ b/pkg/lightning/restore/restore_test.go @@ -37,7 +37,6 @@ import ( kv "github.com/pingcap/br/pkg/lightning/backend" "github.com/pingcap/br/pkg/lightning/checkpoints" - . "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/glue" @@ -67,7 +66,7 @@ func (s *restoreSuite) TestNewTableRestore(c *C) { p := parser.New() se := tmock.NewContext() - dbInfo := &TidbDBInfo{Name: "mockdb", Tables: map[string]*TidbTableInfo{}} + dbInfo := &checkpoints.TidbDBInfo{Name: "mockdb", Tables: map[string]*checkpoints.TidbTableInfo{}} for i, tc := range testCases { node, err := p.ParseOneStmt(tc.createStmt, "utf8mb4", "utf8mb4_bin") c.Assert(err, IsNil) @@ -75,7 +74,7 @@ func (s *restoreSuite) TestNewTableRestore(c *C) { c.Assert(err, IsNil) tableInfo.State = model.StatePublic - dbInfo.Tables[tc.name] = &TidbTableInfo{ + dbInfo.Tables[tc.name] = &checkpoints.TidbTableInfo{ Name: tc.name, Core: tableInfo, } @@ -84,23 +83,23 @@ func (s *restoreSuite) TestNewTableRestore(c *C) { for _, tc := range testCases { tableInfo := dbInfo.Tables[tc.name] tableName := common.UniqueTable("mockdb", tableInfo.Name) - tr, err := NewTableRestore(tableName, nil, dbInfo, tableInfo, &TableCheckpoint{}) + tr, err := NewTableRestore(tableName, nil, dbInfo, tableInfo, &checkpoints.TableCheckpoint{}) c.Assert(tr, NotNil) c.Assert(err, IsNil) } } func (s *restoreSuite) TestNewTableRestoreFailure(c *C) { - tableInfo := &TidbTableInfo{ + tableInfo := &checkpoints.TidbTableInfo{ Name: "failure", Core: &model.TableInfo{}, } - dbInfo := &TidbDBInfo{Name: "mockdb", Tables: map[string]*TidbTableInfo{ + dbInfo := &checkpoints.TidbDBInfo{Name: "mockdb", Tables: map[string]*checkpoints.TidbTableInfo{ "failure": tableInfo, }} tableName := common.UniqueTable("mockdb", "failure") - _, err := NewTableRestore(tableName, nil, dbInfo, tableInfo, &TableCheckpoint{}) + _, err := NewTableRestore(tableName, nil, dbInfo, tableInfo, &checkpoints.TableCheckpoint{}) c.Assert(err, ErrorMatches, `failed to tables\.TableFromMeta.*`) } @@ -108,8 +107,8 @@ func (s *restoreSuite) TestErrorSummaries(c *C) { logger, buffer := log.MakeTestLogger() es := makeErrorSummaries(logger) - es.record("first", errors.New("a1 error"), CheckpointStatusAnalyzed) - es.record("second", errors.New("b2 error"), CheckpointStatusAllWritten) + es.record("first", errors.New("a1 error"), checkpoints.CheckpointStatusAnalyzed) + es.record("second", errors.New("b2 error"), checkpoints.CheckpointStatusAllWritten) es.emitLog() lines := buffer.Lines() @@ -208,8 +207,8 @@ type tableRestoreSuiteBase struct { tr *TableRestore cfg *config.Config - tableInfo *TidbTableInfo - dbInfo *TidbDBInfo + tableInfo *checkpoints.TidbTableInfo + dbInfo *checkpoints.TidbDBInfo tableMeta *mydump.MDTableMeta store storage.ExternalStorage @@ -238,10 +237,10 @@ func (s *tableRestoreSuiteBase) SetUpSuite(c *C) { c.Assert(err, IsNil) core.State = model.StatePublic - s.tableInfo = &TidbTableInfo{Name: "table", DB: "db", Core: core} - s.dbInfo = &TidbDBInfo{ + s.tableInfo = &checkpoints.TidbTableInfo{Name: "table", DB: "db", Core: core} + s.dbInfo = &checkpoints.TidbDBInfo{ Name: "db", - Tables: map[string]*TidbTableInfo{"table": s.tableInfo}, + Tables: map[string]*checkpoints.TidbTableInfo{"table": s.tableInfo}, } // Write some sample SQL dump @@ -261,28 +260,50 @@ func (s *tableRestoreSuiteBase) SetUpSuite(c *C) { fakeDataPath := filepath.Join(fakeDataDir, fakeFileName) err = ioutil.WriteFile(fakeDataPath, fakeDataFilesContent, 0o644) c.Assert(err, IsNil) - fakeDataFiles = append(fakeDataFiles, mydump.FileInfo{TableName: filter.Table{"db", "table"}, FileMeta: mydump.SourceFileMeta{Path: fakeFileName, Type: mydump.SourceTypeSQL, SortKey: fmt.Sprintf("%d", i), FileSize: 37}}) + fakeDataFiles = append(fakeDataFiles, mydump.FileInfo{ + TableName: filter.Table{Schema: "db", Name: "table"}, + FileMeta: mydump.SourceFileMeta{ + Path: fakeFileName, + Type: mydump.SourceTypeSQL, + SortKey: fmt.Sprintf("%d", i), + FileSize: 37, + }, + }) } fakeCsvContent := []byte("1,2,3\r\n4,5,6\r\n") csvName := "db.table.99.csv" err = ioutil.WriteFile(filepath.Join(fakeDataDir, csvName), fakeCsvContent, 0o644) c.Assert(err, IsNil) - fakeDataFiles = append(fakeDataFiles, mydump.FileInfo{TableName: filter.Table{"db", "table"}, FileMeta: mydump.SourceFileMeta{Path: csvName, Type: mydump.SourceTypeCSV, SortKey: "99", FileSize: 14}}) + fakeDataFiles = append(fakeDataFiles, mydump.FileInfo{ + TableName: filter.Table{Schema: "db", Name: "table"}, + FileMeta: mydump.SourceFileMeta{ + Path: csvName, + Type: mydump.SourceTypeCSV, + SortKey: "99", + FileSize: 14, + }, + }) s.tableMeta = &mydump.MDTableMeta{ - DB: "db", - Name: "table", - TotalSize: 222, - SchemaFile: mydump.FileInfo{TableName: filter.Table{Schema: "db", Name: "table"}, FileMeta: mydump.SourceFileMeta{Path: "db.table-schema.sql", Type: mydump.SourceTypeTableSchema}}, - DataFiles: fakeDataFiles, + DB: "db", + Name: "table", + TotalSize: 222, + SchemaFile: mydump.FileInfo{ + TableName: filter.Table{Schema: "db", Name: "table"}, + FileMeta: mydump.SourceFileMeta{ + Path: "db.table-schema.sql", + Type: mydump.SourceTypeTableSchema, + }, + }, + DataFiles: fakeDataFiles, } } func (s *tableRestoreSuiteBase) SetUpTest(c *C) { // Collect into the test TableRestore structure var err error - s.tr, err = NewTableRestore("`db`.`table`", s.tableMeta, s.dbInfo, s.tableInfo, &TableCheckpoint{}) + s.tr, err = NewTableRestore("`db`.`table`", s.tableMeta, s.dbInfo, s.tableInfo, &checkpoints.TableCheckpoint{}) c.Assert(err, IsNil) s.cfg = config.NewConfig() @@ -291,25 +312,28 @@ func (s *tableRestoreSuiteBase) SetUpTest(c *C) { } func (s *tableRestoreSuite) TestPopulateChunks(c *C) { - failpoint.Enable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp", "return(1234567897)") - defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp") + _ = failpoint.Enable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp", "return(1234567897)") + defer func() { + _ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp") + }() - cp := &TableCheckpoint{ - Engines: make(map[int32]*EngineCheckpoint), + cp := &checkpoints.TableCheckpoint{ + Engines: make(map[int32]*checkpoints.EngineCheckpoint), } - rc := &RestoreController{cfg: s.cfg, ioWorkers: worker.NewPool(context.Background(), 1, "io"), store: s.store} + rc := &Controller{cfg: s.cfg, ioWorkers: worker.NewPool(context.Background(), 1, "io"), store: s.store} err := s.tr.populateChunks(context.Background(), rc, cp) c.Assert(err, IsNil) - c.Assert(cp.Engines, DeepEquals, map[int32]*EngineCheckpoint{ + //nolint:dupl // false positive. + c.Assert(cp.Engines, DeepEquals, map[int32]*checkpoints.EngineCheckpoint{ -1: { - Status: CheckpointStatusLoaded, + Status: checkpoints.CheckpointStatusLoaded, }, 0: { - Status: CheckpointStatusLoaded, - Chunks: []*ChunkCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, + Chunks: []*checkpoints.ChunkCheckpoint{ { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[0].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[0].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[0].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -320,7 +344,7 @@ func (s *tableRestoreSuite) TestPopulateChunks(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[1].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[1].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[1].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -331,7 +355,7 @@ func (s *tableRestoreSuite) TestPopulateChunks(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[2].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[2].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[2].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -344,10 +368,10 @@ func (s *tableRestoreSuite) TestPopulateChunks(c *C) { }, }, 1: { - Status: CheckpointStatusLoaded, - Chunks: []*ChunkCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, + Chunks: []*checkpoints.ChunkCheckpoint{ { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[3].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[3].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[3].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -358,7 +382,7 @@ func (s *tableRestoreSuite) TestPopulateChunks(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[4].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[4].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[4].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -369,7 +393,7 @@ func (s *tableRestoreSuite) TestPopulateChunks(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[5].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[5].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[5].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -382,10 +406,10 @@ func (s *tableRestoreSuite) TestPopulateChunks(c *C) { }, }, 2: { - Status: CheckpointStatusLoaded, - Chunks: []*ChunkCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, + Chunks: []*checkpoints.ChunkCheckpoint{ { - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[6].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[6].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[6].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -436,7 +460,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { err := ioutil.WriteFile(filepath.Join(fakeDataDir, csvName), []byte(s), 0o644) c.Assert(err, IsNil) fakeDataFiles = append(fakeDataFiles, mydump.FileInfo{ - TableName: filter.Table{"db", "table"}, + TableName: filter.Table{Schema: "db", Name: "table"}, FileMeta: mydump.SourceFileMeta{Path: csvName, Type: mydump.SourceTypeCSV, SortKey: fmt.Sprintf("%02d", i), FileSize: int64(len(s))}, }) total += len(s) @@ -449,11 +473,13 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { DataFiles: fakeDataFiles, } - failpoint.Enable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp", "return(1234567897)") - defer failpoint.Disable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp") + _ = failpoint.Enable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp", "return(1234567897)") + defer func() { + _ = failpoint.Disable("github.com/pingcap/br/pkg/lightning/restore/PopulateChunkTimestamp") + }() - cp := &TableCheckpoint{ - Engines: make(map[int32]*EngineCheckpoint), + cp := &checkpoints.TableCheckpoint{ + Engines: make(map[int32]*checkpoints.EngineCheckpoint), } cfg := config.NewConfig() @@ -462,21 +488,21 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { cfg.Mydumper.CSV.Header = true cfg.Mydumper.StrictFormat = true - rc := &RestoreController{cfg: cfg, ioWorkers: worker.NewPool(context.Background(), 1, "io"), store: store} + rc := &Controller{cfg: cfg, ioWorkers: worker.NewPool(context.Background(), 1, "io"), store: store} - tr, err := NewTableRestore("`db`.`table`", tableMeta, s.dbInfo, s.tableInfo, &TableCheckpoint{}) + tr, err := NewTableRestore("`db`.`table`", tableMeta, s.dbInfo, s.tableInfo, &checkpoints.TableCheckpoint{}) c.Assert(err, IsNil) c.Assert(tr.populateChunks(context.Background(), rc, cp), IsNil) - c.Assert(cp.Engines, DeepEquals, map[int32]*EngineCheckpoint{ + c.Assert(cp.Engines, DeepEquals, map[int32]*checkpoints.EngineCheckpoint{ -1: { - Status: CheckpointStatusLoaded, + Status: checkpoints.CheckpointStatusLoaded, }, 0: { - Status: CheckpointStatusLoaded, - Chunks: []*ChunkCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, + Chunks: []*checkpoints.ChunkCheckpoint{ { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[0].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[0].FileMeta.Path, Offset: 0}, FileMeta: tableMeta.DataFiles[0].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -487,7 +513,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[1].FileMeta.Path, Offset: 0}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[1].FileMeta.Path, Offset: 0}, FileMeta: tableMeta.DataFiles[1].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -498,7 +524,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[2].FileMeta.Path, Offset: 6}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[2].FileMeta.Path, Offset: 6}, FileMeta: tableMeta.DataFiles[2].FileMeta, ColumnPermutation: []int{0, 1, 2, -1}, Chunk: mydump.Chunk{ @@ -512,7 +538,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[2].FileMeta.Path, Offset: 52}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[2].FileMeta.Path, Offset: 52}, FileMeta: tableMeta.DataFiles[2].FileMeta, ColumnPermutation: []int{0, 1, 2, -1}, Chunk: mydump.Chunk{ @@ -525,7 +551,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[3].FileMeta.Path, Offset: 6}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[3].FileMeta.Path, Offset: 6}, FileMeta: tableMeta.DataFiles[3].FileMeta, ColumnPermutation: []int{1, 2, 0, -1}, Chunk: mydump.Chunk{ @@ -540,10 +566,10 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { }, }, 1: { - Status: CheckpointStatusLoaded, - Chunks: []*ChunkCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, + Chunks: []*checkpoints.ChunkCheckpoint{ { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[3].FileMeta.Path, Offset: 48}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[3].FileMeta.Path, Offset: 48}, FileMeta: tableMeta.DataFiles[3].FileMeta, ColumnPermutation: []int{1, 2, 0, -1}, Chunk: mydump.Chunk{ @@ -556,7 +582,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[3].FileMeta.Path, Offset: 101}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[3].FileMeta.Path, Offset: 101}, FileMeta: tableMeta.DataFiles[3].FileMeta, ColumnPermutation: []int{1, 2, 0, -1}, Chunk: mydump.Chunk{ @@ -569,7 +595,7 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { Timestamp: 1234567897, }, { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[4].FileMeta.Path, Offset: 4}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[4].FileMeta.Path, Offset: 4}, FileMeta: tableMeta.DataFiles[4].FileMeta, ColumnPermutation: []int{-1, 0, 1, -1}, Chunk: mydump.Chunk{ @@ -584,10 +610,10 @@ func (s *tableRestoreSuite) TestPopulateChunksCSVHeader(c *C) { }, }, 2: { - Status: CheckpointStatusLoaded, - Chunks: []*ChunkCheckpoint{ + Status: checkpoints.CheckpointStatusLoaded, + Chunks: []*checkpoints.ChunkCheckpoint{ { - Key: ChunkCheckpointKey{Path: tableMeta.DataFiles[4].FileMeta.Path, Offset: 59}, + Key: checkpoints.ChunkCheckpointKey{Path: tableMeta.DataFiles[4].FileMeta.Path, Offset: 59}, FileMeta: tableMeta.DataFiles[4].FileMeta, ColumnPermutation: []int{-1, 0, 1, -1}, Chunk: mydump.Chunk{ @@ -619,7 +645,7 @@ func (s *tableRestoreSuite) TestGetColumnsNames(c *C) { } func (s *tableRestoreSuite) TestInitializeColumns(c *C) { - ccp := &ChunkCheckpoint{} + ccp := &checkpoints.ChunkCheckpoint{} c.Assert(s.tr.initializeColumns(nil, ccp), IsNil) c.Assert(ccp.ColumnPermutation, DeepEquals, []int{0, 1, 2, -1}) @@ -726,7 +752,7 @@ func (s *tableRestoreSuite) TestImportKVSuccess(c *C) { importer := kv.MakeBackend(mockBackend) chptCh := make(chan saveCp) defer close(chptCh) - rc := &RestoreController{saveCpCh: chptCh} + rc := &Controller{saveCpCh: chptCh} go func() { for range chptCh { } @@ -758,7 +784,7 @@ func (s *tableRestoreSuite) TestImportKVFailure(c *C) { importer := kv.MakeBackend(mockBackend) chptCh := make(chan saveCp) defer close(chptCh) - rc := &RestoreController{saveCpCh: chptCh} + rc := &Controller{saveCpCh: chptCh} go func() { for range chptCh { } @@ -791,7 +817,7 @@ func (s *tableRestoreSuite) TestCheckRequirements(c *C) { mockBackend.EXPECT(). CheckRequirements(ctx). Return(errors.Annotate(context.Canceled, "fake check requirement error")) - rc := &RestoreController{ + rc := &Controller{ cfg: &config.Config{App: config.Lightning{CheckRequirements: true}}, backend: backend, } @@ -800,6 +826,88 @@ func (s *tableRestoreSuite) TestCheckRequirements(c *C) { c.Assert(err, ErrorMatches, "fake check requirement error.*") } +<<<<<<< HEAD +======= +func (s *tableRestoreSuite) TestTableRestoreMetrics(c *C) { + controller := gomock.NewController(c) + defer controller.Finish() + + chunkPendingBase := metric.ReadCounter(metric.ChunkCounter.WithLabelValues(metric.ChunkStatePending)) + chunkFinishedBase := metric.ReadCounter(metric.ChunkCounter.WithLabelValues(metric.ChunkStatePending)) + engineFinishedBase := metric.ReadCounter(metric.ProcessedEngineCounter.WithLabelValues("imported", metric.TableResultSuccess)) + tableFinishedBase := metric.ReadCounter(metric.TableCounter.WithLabelValues("index_imported", metric.TableResultSuccess)) + + ctx := context.Background() + chptCh := make(chan saveCp) + defer close(chptCh) + cfg := config.NewConfig() + cfg.Mydumper.BatchSize = 1 + cfg.PostRestore.Checksum = config.OpLevelOff + + cfg.Checkpoint.Enable = false + cfg.TiDB.Host = "127.0.0.1" + cfg.TiDB.StatusPort = 10080 + cfg.TiDB.Port = 4000 + cfg.TiDB.PdAddr = "127.0.0.1:2379" + + cfg.Mydumper.SourceDir = "." + cfg.Mydumper.CSV.Header = false + tls, err := cfg.ToTLS() + c.Assert(err, IsNil) + + err = cfg.Adjust(ctx) + c.Assert(err, IsNil) + + cpDB := checkpoints.NewNullCheckpointsDB() + rc := &Controller{ + cfg: cfg, + dbMetas: []*mydump.MDDatabaseMeta{ + { + Name: s.tableInfo.DB, + Tables: []*mydump.MDTableMeta{s.tableMeta}, + }, + }, + dbInfos: map[string]*checkpoints.TidbDBInfo{ + s.tableInfo.DB: s.dbInfo, + }, + tableWorkers: worker.NewPool(ctx, 6, "table"), + indexWorkers: worker.NewPool(ctx, 2, "index"), + ioWorkers: worker.NewPool(ctx, 5, "io"), + regionWorkers: worker.NewPool(ctx, 10, "region"), + checksumWorks: worker.NewPool(ctx, 2, "region"), + saveCpCh: chptCh, + pauser: DeliverPauser, + backend: noop.NewNoopBackend(), + tidbGlue: mock.NewMockGlue(controller), + errorSummaries: makeErrorSummaries(log.L()), + tls: tls, + checkpointsDB: cpDB, + closedEngineLimit: worker.NewPool(ctx, 1, "closed_engine"), + store: s.store, + } + go func() { + for range chptCh { + } + }() + + web.BroadcastInitProgress(rc.dbMetas) + + err = rc.restoreTables(ctx) + c.Assert(err, IsNil) + + chunkPending := metric.ReadCounter(metric.ChunkCounter.WithLabelValues(metric.ChunkStatePending)) + chunkFinished := metric.ReadCounter(metric.ChunkCounter.WithLabelValues(metric.ChunkStatePending)) + c.Assert(chunkPending-chunkPendingBase, Equals, float64(7)) + c.Assert(chunkFinished-chunkFinishedBase, Equals, chunkPending) + + engineFinished := metric.ReadCounter(metric.ProcessedEngineCounter.WithLabelValues("imported", metric.TableResultSuccess)) + c.Assert(engineFinished-engineFinishedBase, Equals, float64(8)) + + tableFinished := metric.ReadCounter(metric.TableCounter.WithLabelValues("index_imported", metric.TableResultSuccess)) + c.Assert(tableFinished-tableFinishedBase, Equals, float64(1)) +} + +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) var _ = Suite(&chunkRestoreSuite{}) type chunkRestoreSuite struct { @@ -813,8 +921,8 @@ func (s *chunkRestoreSuite) SetUpTest(c *C) { ctx := context.Background() w := worker.NewPool(ctx, 5, "io") - chunk := ChunkCheckpoint{ - Key: ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[1].FileMeta.Path, Offset: 0}, + chunk := checkpoints.ChunkCheckpoint{ + Key: checkpoints.ChunkCheckpointKey{Path: s.tr.tableMeta.DataFiles[1].FileMeta.Path, Offset: 0}, FileMeta: s.tr.tableMeta.DataFiles[1].FileMeta, Chunk: mydump.Chunk{ Offset: 0, @@ -834,7 +942,11 @@ func (s *chunkRestoreSuite) TearDownTest(c *C) { } func (s *chunkRestoreSuite) TestDeliverLoopCancel(c *C) { +<<<<<<< HEAD rc := &RestoreController{backend: kv.NewMockImporter(nil, "")} +======= + rc := &Controller{backend: importer.NewMockImporter(nil, "")} +>>>>>>> 4c77b100... lightning: Fix lints for lightning (#766) ctx, cancel := context.WithCancel(context.Background()) kvsCh := make(chan []deliveredKVs) @@ -873,7 +985,7 @@ func (s *chunkRestoreSuite) TestDeliverLoopEmptyData(c *C) { // Deliver nothing. cfg := &config.Config{} - rc := &RestoreController{cfg: cfg, backend: importer} + rc := &Controller{cfg: cfg, backend: importer} kvsCh := make(chan []deliveredKVs, 1) kvsCh <- []deliveredKVs{} @@ -966,7 +1078,7 @@ func (s *chunkRestoreSuite) TestDeliverLoop(c *C) { }() cfg := &config.Config{} - rc := &RestoreController{cfg: cfg, saveCpCh: saveCpCh, backend: importer} + rc := &Controller{cfg: cfg, saveCpCh: saveCpCh, backend: importer} _, err = s.cr.deliverLoop(ctx, kvsCh, s.tr, 0, dataWriter, indexWriter, rc) c.Assert(err, IsNil) @@ -986,7 +1098,7 @@ func (s *chunkRestoreSuite) TestEncodeLoop(c *C) { }) c.Assert(err, IsNil) cfg := config.NewConfig() - rc := &RestoreController{pauser: DeliverPauser, cfg: cfg} + rc := &Controller{pauser: DeliverPauser, cfg: cfg} _, _, err = s.cr.encodeLoop(ctx, kvsCh, s.tr, s.tr.logger, kvEncoder, deliverCompleteCh, rc) c.Assert(err, IsNil) c.Assert(kvsCh, HasLen, 2) @@ -1013,7 +1125,7 @@ func (s *chunkRestoreSuite) TestEncodeLoopCanceled(c *C) { go cancel() cfg := config.NewConfig() - rc := &RestoreController{pauser: DeliverPauser, cfg: cfg} + rc := &Controller{pauser: DeliverPauser, cfg: cfg} _, _, err = s.cr.encodeLoop(ctx, kvsCh, s.tr, s.tr.logger, kvEncoder, deliverCompleteCh, rc) c.Assert(errors.Cause(err), Equals, context.Canceled) c.Assert(kvsCh, HasLen, 0) @@ -1033,7 +1145,7 @@ func (s *chunkRestoreSuite) TestEncodeLoopForcedError(c *C) { s.cr.parser.Close() cfg := config.NewConfig() - rc := &RestoreController{pauser: DeliverPauser, cfg: cfg} + rc := &Controller{pauser: DeliverPauser, cfg: cfg} _, _, err = s.cr.encodeLoop(ctx, kvsCh, s.tr, s.tr.logger, kvEncoder, deliverCompleteCh, rc) c.Assert(err, ErrorMatches, `in file .*[/\\]?db\.table\.2\.sql:0 at offset 0:.*file already closed`) c.Assert(kvsCh, HasLen, 0) @@ -1055,7 +1167,7 @@ func (s *chunkRestoreSuite) TestEncodeLoopDeliverErrored(c *C) { } }() cfg := config.NewConfig() - rc := &RestoreController{pauser: DeliverPauser, cfg: cfg} + rc := &Controller{pauser: DeliverPauser, cfg: cfg} _, _, err = s.cr.encodeLoop(ctx, kvsCh, s.tr, s.tr.logger, kvEncoder, deliverCompleteCh, rc) c.Assert(err, ErrorMatches, "fake deliver error") c.Assert(kvsCh, HasLen, 0) @@ -1072,7 +1184,7 @@ func (s *chunkRestoreSuite) TestEncodeLoopColumnsMismatch(c *C) { ctx := context.Background() cfg := config.NewConfig() - rc := &RestoreController{pauser: DeliverPauser, cfg: cfg} + rc := &Controller{pauser: DeliverPauser, cfg: cfg} reader, err := store.Open(ctx, fileName) c.Assert(err, IsNil) @@ -1144,7 +1256,7 @@ func (s *chunkRestoreSuite) TestRestore(c *C) { // Now actually start the restore loop. saveCpCh := make(chan saveCp, 2) - err = s.cr.restore(ctx, s.tr, 0, dataWriter, indexWriter, &RestoreController{ + err = s.cr.restore(ctx, s.tr, 0, dataWriter, indexWriter, &Controller{ cfg: s.cfg, saveCpCh: saveCpCh, backend: importer, @@ -1158,7 +1270,7 @@ var _ = Suite(&restoreSchemaSuite{}) type restoreSchemaSuite struct { ctx context.Context - rc *RestoreController + rc *Controller controller *gomock.Controller } @@ -1200,7 +1312,7 @@ func (s *restoreSchemaSuite) SetUpSuite(c *C) { config.App.RegionConcurrency = 8 mydumpLoader, err := mydump.NewMyDumpLoaderWithStore(ctx, config, store) c.Assert(err, IsNil) - s.rc = &RestoreController{ + s.rc = &Controller{ cfg: config, store: store, dbMetas: mydumpLoader.GetDatabases(), @@ -1208,6 +1320,7 @@ func (s *restoreSchemaSuite) SetUpSuite(c *C) { } } +//nolint:interfacer // change test case signature might cause Check failed to find this test case? func (s *restoreSchemaSuite) SetUpTest(c *C) { s.controller, s.ctx = gomock.WithContext(context.Background(), c) mockBackend := mock.NewMockBackend(s.controller) diff --git a/pkg/lightning/restore/tidb.go b/pkg/lightning/restore/tidb.go index 18b7713a4..3ffc05fdf 100644 --- a/pkg/lightning/restore/tidb.go +++ b/pkg/lightning/restore/tidb.go @@ -31,7 +31,7 @@ import ( "github.com/pingcap/br/pkg/lightning/glue" - . "github.com/pingcap/br/pkg/lightning/checkpoints" + "github.com/pingcap/br/pkg/lightning/checkpoints" "github.com/pingcap/br/pkg/lightning/common" "github.com/pingcap/br/pkg/lightning/config" "github.com/pingcap/br/pkg/lightning/log" @@ -236,17 +236,17 @@ func LoadSchemaInfo( ctx context.Context, schemas []*mydump.MDDatabaseMeta, getTables func(context.Context, string) ([]*model.TableInfo, error), -) (map[string]*TidbDBInfo, error) { - result := make(map[string]*TidbDBInfo, len(schemas)) +) (map[string]*checkpoints.TidbDBInfo, error) { + result := make(map[string]*checkpoints.TidbDBInfo, len(schemas)) for _, schema := range schemas { tables, err := getTables(ctx, schema.Name) if err != nil { return nil, err } - dbInfo := &TidbDBInfo{ + dbInfo := &checkpoints.TidbDBInfo{ Name: schema.Name, - Tables: make(map[string]*TidbTableInfo), + Tables: make(map[string]*checkpoints.TidbTableInfo), } for _, tbl := range tables { @@ -260,7 +260,7 @@ func LoadSchemaInfo( if err != nil { return nil, errors.Trace(err) } - tableInfo := &TidbTableInfo{ + tableInfo := &checkpoints.TidbTableInfo{ ID: tbl.ID, DB: schema.Name, Name: tableName, diff --git a/pkg/lightning/restore/tidb_test.go b/pkg/lightning/restore/tidb_test.go index fab65d679..c9f7ff021 100644 --- a/pkg/lightning/restore/tidb_test.go +++ b/pkg/lightning/restore/tidb_test.go @@ -15,7 +15,6 @@ package restore import ( "context" - "net/http" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -37,10 +36,9 @@ import ( var _ = Suite(&tidbSuite{}) type tidbSuite struct { - mockDB sqlmock.Sqlmock - handler http.Handler - timgr *TiDBManager - tiGlue glue.Glue + mockDB sqlmock.Sqlmock + timgr *TiDBManager + tiGlue glue.Glue } func TestTiDB(t *testing.T) { diff --git a/pkg/lightning/tikv/tikv_test.go b/pkg/lightning/tikv/tikv_test.go new file mode 100644 index 000000000..aa5467722 --- /dev/null +++ b/pkg/lightning/tikv/tikv_test.go @@ -0,0 +1,252 @@ +package tikv_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "sync" + + "github.com/coreos/go-semver/semver" + . "github.com/pingcap/check" + "github.com/pingcap/kvproto/pkg/import_sstpb" + + "github.com/pingcap/br/pkg/lightning/common" + kv "github.com/pingcap/br/pkg/lightning/tikv" +) + +type tikvSuite struct{} + +var _ = Suite(&tikvSuite{}) + +var ( + // Samples from importer backend for testing the Check***Version functions. + // No need keep these versions in sync. + requiredMinPDVersion = *semver.New("2.1.0") + requiredMinTiKVVersion = *semver.New("2.1.0") + requiredMaxPDVersion = *semver.New("6.0.0") + requiredMaxTiKVVersion = *semver.New("6.0.0") +) + +func (s *tikvSuite) TestForAllStores(c *C) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, err := w.Write([]byte(` + { + "count": 5, + "stores": [ + { + "store": { + "id": 1, + "address": "127.0.0.1:20160", + "version": "3.0.0-beta.1", + "state_name": "Up" + }, + "status": {} + }, + { + "store": { + "id": 2, + "address": "127.0.0.1:20161", + "version": "3.0.0-rc.1", + "state_name": "Down" + }, + "status": {} + }, + { + "store": { + "id": 3, + "address": "127.0.0.1:20162", + "version": "3.0.0-rc.2", + "state_name": "Disconnected" + }, + "status": {} + }, + { + "store": { + "id": 4, + "address": "127.0.0.1:20163", + "version": "3.0.0", + "state_name": "Tombstone" + }, + "status": {} + }, + { + "store": { + "id": 5, + "address": "127.0.0.1:20164", + "version": "3.0.1", + "state_name": "Offline" + }, + "status": {} + } + ] + } + `)) + c.Assert(err, IsNil) + })) + defer server.Close() + + ctx := context.Background() + var ( + allStoresLock sync.Mutex + allStores []*kv.Store + ) + tls := common.NewTLSFromMockServer(server) + err := kv.ForAllStores(ctx, tls, kv.StoreStateDown, func(c2 context.Context, store *kv.Store) error { + allStoresLock.Lock() + allStores = append(allStores, store) + allStoresLock.Unlock() + return nil + }) + c.Assert(err, IsNil) + + sort.Slice(allStores, func(i, j int) bool { return allStores[i].Address < allStores[j].Address }) + + c.Assert(allStores, DeepEquals, []*kv.Store{ + { + Address: "127.0.0.1:20160", + Version: "3.0.0-beta.1", + State: kv.StoreStateUp, + }, + { + Address: "127.0.0.1:20161", + Version: "3.0.0-rc.1", + State: kv.StoreStateDown, + }, + { + Address: "127.0.0.1:20162", + Version: "3.0.0-rc.2", + State: kv.StoreStateDisconnected, + }, + { + Address: "127.0.0.1:20164", + Version: "3.0.1", + State: kv.StoreStateOffline, + }, + }) +} + +func (s *tikvSuite) TestFetchModeFromMetrics(c *C) { + testCases := []struct { + metrics string + mode import_sstpb.SwitchMode + isErr bool + }{ + { + metrics: `tikv_config_rocksdb{cf="default",name="hard_pending_compaction_bytes_limit"} 274877906944`, + mode: import_sstpb.SwitchMode_Normal, + }, + { + metrics: `tikv_config_rocksdb{cf="default",name="hard_pending_compaction_bytes_limit"} 0`, + mode: import_sstpb.SwitchMode_Import, + }, + { + metrics: ``, + isErr: true, + }, + } + + for _, tc := range testCases { + comment := Commentf("test case '%s'", tc.metrics) + mode, err := kv.FetchModeFromMetrics(tc.metrics) + if tc.isErr { + c.Assert(err, NotNil, comment) + } else { + c.Assert(err, IsNil, comment) + c.Assert(mode, Equals, tc.mode, comment) + } + } +} + +func (s *tikvSuite) TestCheckPDVersion(c *C) { + var version string + ctx := context.Background() + + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + c.Assert(req.URL.Path, Equals, "/pd/api/v1/version") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(version)) + c.Assert(err, IsNil) + })) + mockURL, err := url.Parse(mockServer.URL) + c.Assert(err, IsNil) + + tls := common.NewTLSFromMockServer(mockServer) + + version = `{ + "version": "v4.0.0-rc.2-451-g760fb650" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), IsNil) + + version = `{ + "version": "v4.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), IsNil) + + version = `{ + "version": "v9999.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") + + version = `{ + "version": "v6.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") + + version = `{ + "version": "v6.0.0-beta" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too new.*") + + version = `{ + "version": "v1.0.0" +}` + c.Assert(kv.CheckPDVersion(ctx, tls, mockURL.Host, requiredMinPDVersion, requiredMaxPDVersion), ErrorMatches, "PD version too old.*") +} + +func (s *tikvSuite) TestCheckTiKVVersion(c *C) { + var versions []string + ctx := context.Background() + + mockServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + c.Assert(req.URL.Path, Equals, "/pd/api/v1/stores") + w.WriteHeader(http.StatusOK) + + stores := make([]map[string]interface{}, 0, len(versions)) + for i, v := range versions { + stores = append(stores, map[string]interface{}{ + "store": map[string]interface{}{ + "address": fmt.Sprintf("tikv%d.test:20160", i), + "version": v, + }, + }) + } + err := json.NewEncoder(w).Encode(map[string]interface{}{ + "count": len(versions), + "stores": stores, + }) + c.Assert(err, IsNil) + })) + mockURL, err := url.Parse(mockServer.URL) + c.Assert(err, IsNil) + + tls := common.NewTLSFromMockServer(mockServer) + + versions = []string{"4.1.0", "v4.1.0-alpha-9-ga27a7dd"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), IsNil) + + versions = []string{"9999.0.0", "4.0.0"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) + + versions = []string{"4.0.0", "1.0.0"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv1\.test:20160\) version too old.*`) + + versions = []string{"6.0.0"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) + + versions = []string{"6.0.0-beta"} + c.Assert(kv.CheckTiKVVersion(ctx, tls, mockURL.Host, requiredMinTiKVVersion, requiredMaxTiKVVersion), ErrorMatches, `TiKV \(at tikv0\.test:20160\) version too new.*`) +} diff --git a/pkg/lightning/web/progress.go b/pkg/lightning/web/progress.go index 099594dda..d109f7ed3 100644 --- a/pkg/lightning/web/progress.go +++ b/pkg/lightning/web/progress.go @@ -88,9 +88,8 @@ func (cpm *checkpointsMap) marshal(key string) ([]byte, error) { type taskStatus uint8 const ( - taskStatusNotStarted taskStatus = 0 - taskStatusRunning = 1 - taskStatusCompleted = 2 + taskStatusRunning taskStatus = 1 + taskStatusCompleted taskStatus = 2 ) type tableInfo struct {