diff --git a/br/pkg/conn/conn.go b/br/pkg/conn/conn.go index 845774f082..fbe72886c1 100644 --- a/br/pkg/conn/conn.go +++ b/br/pkg/conn/conn.go @@ -108,7 +108,7 @@ func checkStoresAlive(ctx context.Context, // Check live tikv. stores, err := util.GetAllTiKVStores(ctx, pdclient, storeBehavior) if err != nil { - log.Error("fail to get store", zap.Error(err)) + log.Error("failed to get store", zap.Error(err)) return errors.Trace(err) } @@ -149,7 +149,7 @@ func NewMgr( controller, err := pdutil.NewPdController(ctx, pdAddrs, tlsConf, securityOption) if err != nil { - log.Error("fail to create pd controller", zap.Error(err)) + log.Error("failed to create pd controller", zap.Error(err)) return nil, errors.Trace(err) } if checkRequirements { diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index d9cd553a2a..2804013e7d 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -1964,15 +1964,27 @@ func (rc *Client) PreCheckTableTiFlashReplica( return err } for _, table := range tables { - if recorder != nil || - (table.Info.TiFlashReplica != nil && table.Info.TiFlashReplica.Count > tiFlashStoreCount) { - if recorder != nil && table.Info.TiFlashReplica != nil { + if table.Info.TiFlashReplica != nil { + if recorder != nil { recorder.AddTable(table.Info.ID, *table.Info.TiFlashReplica) + log.Info("record tiflash replica for table, to reset it by ddl later", + zap.Stringer("db", table.DB.Name), + zap.Stringer("table", table.Info.Name), + ) + table.Info.TiFlashReplica = nil + } else if table.Info.TiFlashReplica.Count > tiFlashStoreCount { + // we cannot satisfy TiFlash replica in restore cluster. so we should + // set TiFlashReplica to unavailable in tableInfo, to avoid TiDB cannot sense TiFlash and make plan to TiFlash + // see details at https://github.com/pingcap/br/issues/931 + // TODO maybe set table.Info.TiFlashReplica.Count to tiFlashStoreCount, but we need more tests about it. + log.Warn("table does not satisfy tiflash replica requirements, set tiflash replcia to unavaiable", + zap.Stringer("db", table.DB.Name), + zap.Stringer("table", table.Info.Name), + zap.Uint64("expect tiflash replica", table.Info.TiFlashReplica.Count), + zap.Uint64("actual tiflash store", tiFlashStoreCount), + ) + table.Info.TiFlashReplica = nil } - // we cannot satisfy TiFlash replica in restore cluster. so we should - // set TiFlashReplica to unavailable in tableInfo, to avoid TiDB cannot sense TiFlash and make plan to TiFlash - // see details at https://github.com/pingcap/br/issues/931 - table.Info.TiFlashReplica = nil } } return nil diff --git a/br/pkg/restore/client_test.go b/br/pkg/restore/client_test.go index 12787ab503..5db05056a9 100644 --- a/br/pkg/restore/client_test.go +++ b/br/pkg/restore/client_test.go @@ -434,7 +434,7 @@ func TestPreCheckTableTiFlashReplicas(t *testing.T) { } tables[i] = &metautil.Table{ - DB: nil, + DB: &model.DBInfo{Name: model.NewCIStr("test")}, Info: &model.TableInfo{ ID: int64(i), Name: model.NewCIStr("test" + strconv.Itoa(i)), diff --git a/br/pkg/restore/import.go b/br/pkg/restore/import.go index 8ed8778882..b660251026 100644 --- a/br/pkg/restore/import.go +++ b/br/pkg/restore/import.go @@ -557,7 +557,7 @@ func (importer *FileImporter) ImportSSTFiles( continue regionLoop } } - log.Error("download file failed", + log.Warn("download file failed, retry later", logutil.Files(files), logutil.Region(info.Region), logutil.Key("startKey", startKey), @@ -569,7 +569,7 @@ func (importer *FileImporter) ImportSSTFiles( zap.String("file-sample", files[0].Name), zap.Stringer("take", time.Since(start)), logutil.Key("start", files[0].StartKey), logutil.Key("end", files[0].EndKey)) if errIngest := importer.ingest(ctx, info, downloadMetas); errIngest != nil { - log.Error("ingest file failed", + log.Warn("ingest file failed, retry later", logutil.Files(files), logutil.SSTMetas(downloadMetas), logutil.Region(info.Region), @@ -585,7 +585,11 @@ func (importer *FileImporter) ImportSSTFiles( } return nil }, utils.NewImportSSTBackoffer()) - return errors.Trace(err) + if err != nil { + log.Error("import sst file failed after retry, stop the whole progress", logutil.Files(files), zap.Error(err)) + return errors.Trace(err) + } + return nil } func (importer *FileImporter) setDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error { diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index 36adae4e73..a230fb6944 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -888,7 +888,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf break } if resetErr != nil { - log.Error("failed to reset speed limit", zap.Error(resetErr)) + log.Error("failed to reset speed limit, please reset it manually", zap.Error(resetErr)) } }()