From cca35f1604c6927a43c3aaede8e2dd409de8ab72 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 16:57:42 +0800 Subject: [PATCH 1/3] fix panic when pd.GetRegion return nil --- pkg/backup/client.go | 4 ++-- pkg/restore/import.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/backup/client.go b/pkg/backup/client.go index d587db8c5..4e93bf856 100644 --- a/pkg/backup/client.go +++ b/pkg/backup/client.go @@ -546,8 +546,8 @@ func (bc *Client) findRegionLeader( for i := 0; i < 5; i++ { // better backoff. region, err := bc.mgr.GetPDClient().GetRegion(ctx, key) - if err != nil { - log.Error("find leader failed", zap.Error(err)) + if err != nil || region == nil { + log.Error("find leader failed", zap.Error(err), zap.Reflect("region", region)) time.Sleep(time.Millisecond * time.Duration(100*i)) continue } diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 1c0c8e080..2d5b3cb7e 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -289,7 +289,7 @@ func (importer *FileImporter) Import( // Slow path, get region from PD newInfo, errIngest = importer.metaClient.GetRegion( ctx, info.Region.GetStartKey()) - if errIngest != nil { + if errIngest != nil || newInfo == nil { break ingestRetry } } From 94278a057a4de63ef8ff3b72c177d9efcad5ed35 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 17:06:25 +0800 Subject: [PATCH 2/3] check region not found error --- pkg/restore/import.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/restore/import.go b/pkg/restore/import.go index 2d5b3cb7e..bdd264b4b 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -289,9 +289,13 @@ func (importer *FileImporter) Import( // Slow path, get region from PD newInfo, errIngest = importer.metaClient.GetRegion( ctx, info.Region.GetStartKey()) - if errIngest != nil || newInfo == nil { + if errIngest != nil { break ingestRetry } + if newInfo == nil { + errIngest = errors.Annotatef(ErrIngestFailed, "region '%d' not found", info.Region.Id) + continue + } } log.Debug("ingest sst returns not leader error, retry it", utils.ZapRegion(info.Region), From b7e4b5e193d3b536496da0240c599784d2c11668 Mon Sep 17 00:00:00 2001 From: glorv Date: Wed, 9 Sep 2020 17:33:56 +0800 Subject: [PATCH 3/3] continue retry when region not found Signed-off-by: glorv --- pkg/restore/import.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/restore/import.go b/pkg/restore/import.go index bdd264b4b..99da4f53b 100644 --- a/pkg/restore/import.go +++ b/pkg/restore/import.go @@ -292,8 +292,10 @@ func (importer *FileImporter) Import( if errIngest != nil { break ingestRetry } + // do not get region info, wait a second and continue if newInfo == nil { - errIngest = errors.Annotatef(ErrIngestFailed, "region '%d' not found", info.Region.Id) + log.Warn("get region by key return nil", zap.Reflect("region", info.Region)) + time.Sleep(time.Second) continue } }