72 lines
1.7 KiB
Go
72 lines
1.7 KiB
Go
// Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
|
|
|
|
package utils
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.uber.org/multierr"
|
|
)
|
|
|
|
var retryableServerError = []string{
|
|
"server closed",
|
|
"connection refused",
|
|
"connection reset by peer",
|
|
"channel closed",
|
|
"error trying to connect",
|
|
"connection closed before message completed",
|
|
"body write aborted",
|
|
"error during dispatch",
|
|
}
|
|
|
|
// RetryableFunc presents a retryable operation.
|
|
type RetryableFunc func() error
|
|
|
|
// Backoffer implements a backoff policy for retrying operations.
|
|
type Backoffer interface {
|
|
// NextBackoff returns a duration to wait before retrying again
|
|
NextBackoff(err error) time.Duration
|
|
// Attempt returns the remain attempt times
|
|
Attempt() int
|
|
}
|
|
|
|
// WithRetry retries a given operation with a backoff policy.
|
|
//
|
|
// Returns nil if `retryableFunc` succeeded at least once. Otherwise, returns a
|
|
// multierr containing all errors encountered.
|
|
func WithRetry(
|
|
ctx context.Context,
|
|
retryableFunc RetryableFunc,
|
|
backoffer Backoffer,
|
|
) error {
|
|
var allErrors error
|
|
for backoffer.Attempt() > 0 {
|
|
err := retryableFunc()
|
|
if err != nil {
|
|
allErrors = multierr.Append(allErrors, err)
|
|
select {
|
|
case <-ctx.Done():
|
|
return allErrors // nolint:wrapcheck
|
|
case <-time.After(backoffer.NextBackoff(err)):
|
|
}
|
|
} else {
|
|
return nil
|
|
}
|
|
}
|
|
return allErrors // nolint:wrapcheck
|
|
}
|
|
|
|
// MessageIsRetryableStorageError checks whether the message returning from TiKV is retryable ExternalStorageError.
|
|
func MessageIsRetryableStorageError(msg string) bool {
|
|
msgLower := strings.ToLower(msg)
|
|
// UNSAFE! TODO: Add a error type for retryable connection error.
|
|
for _, errStr := range retryableServerError {
|
|
if strings.Contains(msgLower, errStr) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|