75 lines
2.1 KiB
Go
75 lines
2.1 KiB
Go
// Copyright 2016 PingCAP, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package util
|
|
|
|
import (
|
|
"runtime"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
const (
|
|
// DefaultMaxRetries indicates the max retry count.
|
|
DefaultMaxRetries = 30
|
|
// RetryInterval indicates retry interval.
|
|
RetryInterval uint64 = 500
|
|
)
|
|
|
|
// RunWithRetry will run the f with backoff and retry.
|
|
// retryCnt: Max retry count
|
|
// backoff: When run f failed, it will sleep backoff * triedCount time.Millisecond.
|
|
// Function f should have two return value. The first one is an bool which indicate if the err if retryable.
|
|
// The second is if the f meet any error.
|
|
func RunWithRetry(retryCnt int, backoff uint64, f func() (bool, error)) (err error) {
|
|
for i := 1; i <= retryCnt; i++ {
|
|
var retryAble bool
|
|
retryAble, err = f()
|
|
if err == nil || !retryAble {
|
|
return errors.Trace(err)
|
|
}
|
|
sleepTime := time.Duration(backoff*uint64(i)) * time.Millisecond
|
|
time.Sleep(sleepTime)
|
|
}
|
|
return errors.Trace(err)
|
|
}
|
|
|
|
// GetStack gets the stacktrace.
|
|
func GetStack() []byte {
|
|
const size = 4096
|
|
buf := make([]byte, size)
|
|
stackSize := runtime.Stack(buf, false)
|
|
buf = buf[:stackSize]
|
|
return buf
|
|
}
|
|
|
|
// WithRecovery wraps goroutine startup call with force recovery.
|
|
// it will dump current goroutine stack into log if catch any recover result.
|
|
// exec: execute logic function.
|
|
// recoverFn: handler will be called after recover and before dump stack, passing `nil` means noop.
|
|
func WithRecovery(exec func(), recoverFn func(r interface{})) {
|
|
defer func() {
|
|
r := recover()
|
|
if recoverFn != nil {
|
|
recoverFn(r)
|
|
}
|
|
if r != nil {
|
|
buf := GetStack()
|
|
log.Errorf("panic in the recoverable goroutine: %v, stack trace:\n%s", r, buf)
|
|
}
|
|
}()
|
|
exec()
|
|
}
|