database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

restore: fix failed to retry grpc errors (#27423)

Browse Source
This commit is contained in:
山岚
2021-08-20 22:00:01 +08:00
committed by GitHub
parent 7e624445f4
commit 60065dfbee
3 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
br/pkg/restore/backoff.go
View File

@ -56,7 +56,8 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
bo.delayTime = 2 * bo.delayTime
bo.attempt--
} else {
switch errors.Cause(err) { // nolint:errorlint
e := errors.Cause(err)
switch e { // nolint:errorlint
case berrors.ErrKVEpochNotMatch, berrors.ErrKVDownloadFailed, berrors.ErrKVIngestFailed:
bo.delayTime = 2 * bo.delayTime
bo.attempt--
@ -65,7 +66,7 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
bo.delayTime = 0
bo.attempt = 0
default:
switch status.Code(err) {
switch status.Code(e) {
case codes.Unavailable, codes.Aborted:
bo.delayTime = 2 * bo.delayTime
bo.attempt--

4
br/pkg/restore/import.go
View File

@ -317,6 +317,10 @@ func (importer *FileImporter) Import(
log.Debug("failpoint restore-storage-error injected.", zap.String("msg", msg))
e = errors.Annotate(e, msg)
})
failpoint.Inject("restore-gRPC-error", func(_ failpoint.Value) {
log.Warn("the connection to TiKV has been cut by a neko, meow :3")
e = status.Error(codes.Unavailable, "the connection to TiKV has been cut by a neko, meow :3")
})
if e != nil {
remainFiles = remainFiles[i:]
return errors.Trace(e)

2
br/tests/br_full/run.sh
View File

@ -70,7 +70,7 @@ for ct in limit lz4 zstd; do
# restore full
echo "restore with $ct backup start..."
export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/restore-storage-error=1*return(\"connection refused\")"
export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/restore/restore-storage-error=1*return(\"connection refused\");github.com/pingcap/tidb/br/pkg/restore/restore-gRPC-error=1*return(true)"
run_br restore full -s "local://$TEST_DIR/$DB-$ct" --pd $PD_ADDR --ratelimit 1024
export GO_FAILPOINTS=""