From 731117c8dfa2508518c4c41384d51cb1222cc346 Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 30 Nov 2015 11:31:58 +0800 Subject: [PATCH 1/4] tidb: Add full jitter backoff when retry --- session.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/session.go b/session.go index 69e058f1fc..49cdc2afbf 100644 --- a/session.go +++ b/session.go @@ -21,6 +21,8 @@ import ( "bytes" "encoding/json" "fmt" + "math" + "math/rand" "strings" "sync" "sync/atomic" @@ -254,10 +256,26 @@ func (s *session) Retry() error { if (s.maxRetryCnt != unlimitedRetryCnt) && (retryCnt >= s.maxRetryCnt) { return errors.Trace(err) } + expoBackOffFullJitter(retryCnt) } return err } +var ( + // retryBackOffBase is the initial duration, in microsecond, a failed transaction stays dormancy before it retries + retryBackOffBase = 1 + // retryBackOffCap is the max amount of duration, in microsecond, a failed transaction stays dormancy before it retries + retryBackOffCap = 100 +) + +// Implements exponential backoff with full jitter +// See: http://www.awsarchitectureblog.com/2015/03/backoff.html +func expoBackOffFullJitter(attempts int) { + upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts)))) + sleep := time.Duration(rand.Intn(upper)) + time.Sleep(time.Microsecond * sleep) +} + // ExecRestrictedSQL implements SQLHelper interface. // This is used for executing some restricted sql statements. func (s *session) ExecRestrictedSQL(ctx context.Context, sql string) (rset.Recordset, error) { From 64af15bf7c59e903b1771ecc274d8418fdd8717c Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 30 Nov 2015 12:20:48 +0800 Subject: [PATCH 2/4] *: Support backoff and retry limit in RunInNewTxn --- kv/txn.go | 25 +++++++++++++++++++++++-- session.go | 19 +------------------ 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/kv/txn.go b/kv/txn.go index 4e83a5b08a..595c791277 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -14,6 +14,10 @@ package kv import ( + "math" + "math/rand" + "time" + "github.com/juju/errors" "github.com/ngaut/log" "github.com/pingcap/tidb/terror" @@ -36,7 +40,7 @@ func IsRetryableError(err error) bool { // RunInNewTxn will run the f in a new transaction environment. func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error { - for { + for i := 0; i < maxRetryCnt; i++ { txn, err := store.Begin() if err != nil { log.Errorf("RunInNewTxn error - %v", err) @@ -57,14 +61,31 @@ func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) e if retryable && IsRetryableError(err) { log.Warnf("Retry txn %v", txn) txn.Rollback() + ExpoBackOffFullJitter(i) continue } if err != nil { return errors.Trace(err) } - break } return nil } + +var ( + // Max retry count in RunInNewTxn + maxRetryCnt = 100 + // retryBackOffBase is the initial duration, in microsecond, a failed transaction stays dormancy before it retries + retryBackOffBase = 1 + // retryBackOffCap is the max amount of duration, in microsecond, a failed transaction stays dormancy before it retries + retryBackOffCap = 100 +) + +// ExpoBackOffFullJitter Implements exponential backoff with full jitter +// See: http://www.awsarchitectureblog.com/2015/03/backoff.html +func ExpoBackOffFullJitter(attempts int) { + upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts)))) + sleep := time.Duration(rand.Intn(upper)) + time.Sleep(time.Microsecond * sleep) +} diff --git a/session.go b/session.go index 49cdc2afbf..557d9e32f7 100644 --- a/session.go +++ b/session.go @@ -21,8 +21,6 @@ import ( "bytes" "encoding/json" "fmt" - "math" - "math/rand" "strings" "sync" "sync/atomic" @@ -256,26 +254,11 @@ func (s *session) Retry() error { if (s.maxRetryCnt != unlimitedRetryCnt) && (retryCnt >= s.maxRetryCnt) { return errors.Trace(err) } - expoBackOffFullJitter(retryCnt) + kv.ExpoBackOffFullJitter(retryCnt) } return err } -var ( - // retryBackOffBase is the initial duration, in microsecond, a failed transaction stays dormancy before it retries - retryBackOffBase = 1 - // retryBackOffCap is the max amount of duration, in microsecond, a failed transaction stays dormancy before it retries - retryBackOffCap = 100 -) - -// Implements exponential backoff with full jitter -// See: http://www.awsarchitectureblog.com/2015/03/backoff.html -func expoBackOffFullJitter(attempts int) { - upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts)))) - sleep := time.Duration(rand.Intn(upper)) - time.Sleep(time.Microsecond * sleep) -} - // ExecRestrictedSQL implements SQLHelper interface. // This is used for executing some restricted sql statements. func (s *session) ExecRestrictedSQL(ctx context.Context, sql string) (rset.Recordset, error) { From 2c137e3e02b8da44afd3b47aff887764806c03e1 Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 30 Nov 2015 14:48:16 +0800 Subject: [PATCH 3/4] *: Address comment --- kv/txn.go | 6 +++--- session.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kv/txn.go b/kv/txn.go index 595c791277..af9da47356 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -61,7 +61,7 @@ func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) e if retryable && IsRetryableError(err) { log.Warnf("Retry txn %v", txn) txn.Rollback() - ExpoBackOffFullJitter(i) + BackOff(i) continue } if err != nil { @@ -82,9 +82,9 @@ var ( retryBackOffCap = 100 ) -// ExpoBackOffFullJitter Implements exponential backoff with full jitter +// BackOff Implements exponential backoff with full jitter // See: http://www.awsarchitectureblog.com/2015/03/backoff.html -func ExpoBackOffFullJitter(attempts int) { +func BackOff(attempts int) { upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts)))) sleep := time.Duration(rand.Intn(upper)) time.Sleep(time.Microsecond * sleep) diff --git a/session.go b/session.go index 557d9e32f7..0b83dc5d99 100644 --- a/session.go +++ b/session.go @@ -254,7 +254,7 @@ func (s *session) Retry() error { if (s.maxRetryCnt != unlimitedRetryCnt) && (retryCnt >= s.maxRetryCnt) { return errors.Trace(err) } - kv.ExpoBackOffFullJitter(retryCnt) + kv.BackOff(retryCnt) } return err } From f1044308b0bc4006d2d0da331219a73328314378 Mon Sep 17 00:00:00 2001 From: shenli Date: Tue, 1 Dec 2015 10:24:48 +0800 Subject: [PATCH 4/4] *: Add test case for BackOff --- kv/txn.go | 12 +++++++----- kv/txn_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 kv/txn_test.go diff --git a/kv/txn.go b/kv/txn.go index af9da47356..d69fdd685b 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -82,10 +82,12 @@ var ( retryBackOffCap = 100 ) -// BackOff Implements exponential backoff with full jitter -// See: http://www.awsarchitectureblog.com/2015/03/backoff.html -func BackOff(attempts int) { +// BackOff Implements exponential backoff with full jitter. +// Returns real back off time in microsecond. +// See: http://www.awsarchitectureblog.com/2015/03/backoff.html. +func BackOff(attempts int) int { upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts)))) - sleep := time.Duration(rand.Intn(upper)) - time.Sleep(time.Microsecond * sleep) + sleep := time.Duration(rand.Intn(upper)) * time.Microsecond + time.Sleep(sleep) + return int(sleep) } diff --git a/kv/txn_test.go b/kv/txn_test.go new file mode 100644 index 0000000000..4af9d90567 --- /dev/null +++ b/kv/txn_test.go @@ -0,0 +1,40 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv + +import ( + . "github.com/pingcap/check" +) + +var _ = Suite(&testTxnSuite{}) + +type testTxnSuite struct { +} + +func (s *testTxnSuite) SetUpTest(c *C) { +} + +func (s *testTxnSuite) TearDownTest(c *C) { +} + +func (s *testTxnSuite) TestBackOff(c *C) { + mustBackOff(c, 1, 2) + mustBackOff(c, 2, 4) + mustBackOff(c, 3, 8) + mustBackOff(c, 100000, 100) +} + +func mustBackOff(c *C, cnt, sleep int) { + c.Assert(BackOff(cnt), LessEqual, sleep*1000) +}