Merge pull request #658 from pingcap/shenli/retry-backoff

tidb: Add full jitter backoff when retry

kv/txn.go

@@ -14,6 +14,10 @@
 package kv
 
 import (
+	"math"
+	"math/rand"
+	"time"
+
 	"github.com/juju/errors"
 	"github.com/ngaut/log"
 	"github.com/pingcap/tidb/terror"
@@ -36,7 +40,7 @@ func IsRetryableError(err error) bool {
 
 // RunInNewTxn will run the f in a new transaction environment.
 func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error {
-	for {
+	for i := 0; i < maxRetryCnt; i++ {
 		txn, err := store.Begin()
 		if err != nil {
 			log.Errorf("RunInNewTxn error - %v", err)
@@ -57,14 +61,33 @@ func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) e
 		if retryable && IsRetryableError(err) {
 			log.Warnf("Retry txn %v", txn)
 			txn.Rollback()
+			BackOff(i)
 			continue
 		}
 		if err != nil {
 			return errors.Trace(err)
 		}
-
 		break
 	}
 
 	return nil
 }
+
+var (
+	// Max retry count in RunInNewTxn
+	maxRetryCnt = 100
+	// retryBackOffBase is the initial duration, in microsecond, a failed transaction stays dormancy before it retries
+	retryBackOffBase = 1
+	// retryBackOffCap is the max amount of duration, in microsecond, a failed transaction stays dormancy before it retries
+	retryBackOffCap = 100
+)
+
+// BackOff Implements exponential backoff with full jitter.
+// Returns real back off time in microsecond.
+// See: http://www.awsarchitectureblog.com/2015/03/backoff.html.
+func BackOff(attempts int) int {
+	upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts))))
+	sleep := time.Duration(rand.Intn(upper)) * time.Microsecond
+	time.Sleep(sleep)
+	return int(sleep)
+}

kv/txn_test.go

@@ -0,0 +1,40 @@
+// Copyright 2015 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kv
+
+import (
+	. "github.com/pingcap/check"
+)
+
+var _ = Suite(&testTxnSuite{})
+
+type testTxnSuite struct {
+}
+
+func (s *testTxnSuite) SetUpTest(c *C) {
+}
+
+func (s *testTxnSuite) TearDownTest(c *C) {
+}
+
+func (s *testTxnSuite) TestBackOff(c *C) {
+	mustBackOff(c, 1, 2)
+	mustBackOff(c, 2, 4)
+	mustBackOff(c, 3, 8)
+	mustBackOff(c, 100000, 100)
+}
+
+func mustBackOff(c *C, cnt, sleep int) {
+	c.Assert(BackOff(cnt), LessEqual, sleep*1000)
+}

session.go

@@ -254,6 +254,7 @@ func (s *session) Retry() error {
 		if (s.maxRetryCnt != unlimitedRetryCnt) && (retryCnt >= s.maxRetryCnt) {
 			return errors.Trace(err)
 		}
+		kv.BackOff(retryCnt)
 	}
 	return err
 }