database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ebs_backup: added retry for exceeding of quota (#44328)

Browse Source 
close pingcap/tidb#44325
This commit is contained in:
山岚
2023-06-05 15:59:41 +08:00
committed by GitHub
parent d01f0c0431
commit 794002cf45
2 changed files with 24 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
br/pkg/aws/BUILD.bazel
View File

@ -10,6 +10,7 @@ go_library(
"//br/pkg/glue",
"//br/pkg/utils",
"@com_github_aws_aws_sdk_go//aws",
"@com_github_aws_aws_sdk_go//aws/awserr",
"@com_github_aws_aws_sdk_go//aws/session",
"@com_github_aws_aws_sdk_go//service/ec2",
"@com_github_aws_aws_sdk_go//service/ec2/ec2iface",

24
br/pkg/aws/ebs.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
@ -28,6 +29,9 @@ const (
AnnTemporaryVolumeID string = "temporary/volume-id"
EC2K8SClusterNameKey string = "aws:eks:cluster-name"
pollingPendingSnapshotInterval = 30 * time.Second
errCodeTooManyPendingSnapshots = "PendingSnapshotLimitExceeded"
SourcePvcNameKey string = "source/pvcName"
SourceVolumeIdKey string = "source/VolumeId"
SourceTikvNameKey string = "source/TikvName"
@ -202,7 +206,7 @@ func (e *EC2Session) CreateSnapshots(backupInfo *config.EBSBasedBRMeta) (map[str
createSnapshotInput.SetInstanceSpecification(&instanceSpecification)
resp, err := e.ec2.CreateSnapshots(&createSnapshotInput)
resp, err := e.createSnapshotsWithRetry(context.TODO(), &createSnapshotInput)
if err != nil {
return errors.Trace(err)
}
@ -233,6 +237,24 @@ func (e *EC2Session) CreateSnapshots(backupInfo *config.EBSBasedBRMeta) (map[str
return snapIDMap, volAZs, nil
}
func (e *EC2Session) createSnapshotsWithRetry(ctx context.Context, input *ec2.CreateSnapshotsInput) (*ec2.CreateSnapshotsOutput, error) {
for {
res, err := e.ec2.CreateSnapshotsWithContext(ctx, input)
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == errCodeTooManyPendingSnapshots {
log.Warn("the pending snapshots exceeds the limit. waiting...",
zap.String("instance", aws.StringValue(input.InstanceSpecification.InstanceId)),
zap.Strings("volumns", aws.StringValueSlice(input.InstanceSpecification.ExcludeDataVolumeIds)),
)
time.Sleep(pollingPendingSnapshotInterval)
continue
}
if err != nil {
return nil, errors.Annotatef(err, "failed to create snapshot for request %s", input)
}
return res, nil
}
}
func (e *EC2Session) extractSnapProgress(str *string) int64 {
if str == nil {
return 0