From 6bd8c99e7be26fd9d42e0e56244272ee31c9f0b0 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 16:57:42 +0800 Subject: [PATCH 1/3] fix panic when pd.GetRegion return nil --- pkg/backup/client.go | 4 ++-- pkg/restore/import.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/backup/client.go b/pkg/backup/client.go index a4dae42bd..7633cc6b8 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -546,8 +546,8 @@ func (bc *Client) findRegionLeader( for i := 0; i < 5; i++ { // better backoff. region, err := bc.mgr.GetPDClient().GetRegion(ctx, key) - if err != nil { - log.Error("find leader failed", zap.Error(err)) + if err != nil || region == nil { + log.Error("find leader failed", zap.Error(err), zap.Reflect("region", region)) time.Sleep(time.Millisecond * time.Duration(100*i)) continue } diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 0bf2b49dd..84556226b 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -270,7 +270,7 @@ func (importer *FileImporter) Import( // Slow path, get region from PD newInfo, errIngest = importer.metaClient.GetRegion( ctx, info.Region.GetStartKey()) - if errIngest != nil { + if errIngest != nil || newInfo == nil { break ingestRetry } } From f9e052d6593544917eb7cf8addf7f372904d786f Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 17:06:25 +0800 Subject: [PATCH 2/3] check region not found error --- pkg/restore/import.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 84556226b..7684c0a3d 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -270,9 +270,13 @@ func (importer *FileImporter) Import( // Slow path, get region from PD newInfo, errIngest = importer.metaClient.GetRegion( ctx, info.Region.GetStartKey()) - if errIngest != nil || newInfo == nil { + if errIngest != nil { break ingestRetry } + if newInfo == nil { + errIngest = errors.Annotatef(ErrIngestFailed, "region '%d' not found", info.Region.Id) + continue + } } log.Debug("ingest sst returns not leader error, retry it", utils.ZapRegion(info.Region), From 87167d36addc78297f3fbfdeecb15fc2f4d1038d Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 17:33:56 +0800 Subject: [PATCH 3/3] continue retry when region not found Signed-off-by: glorv --- pkg/restore/import.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 7684c0a3d..3fcbcad45 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -273,8 +273,10 @@ func (importer *FileImporter) Import( if errIngest != nil { break ingestRetry } + // do not get region info, wait a second and continue if newInfo == nil { - errIngest = errors.Annotatef(ErrIngestFailed, "region '%d' not found", info.Region.Id) + log.Warn("get region by key return nil", zap.Reflect("region", info.Region)) + time.Sleep(time.Second) continue } }