From 2e1cf26a32a2255e471f8f3e30ef5e5d3b29af71 Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Wed, 16 Dec 2015 18:35:45 +0800 Subject: [PATCH 01/63] optimizer: typeInferrer supports more expression. This is require for implementing prepared statement, because binary protocol depends more on the result field type to decode value, we have to correctly set the result field type. For statement like 'select ?', the type of the field is unknown until we execute the statement with argument, If the field type of parameter marker `?' is not set properly, client will not be able to read the value. --- ast/expressions.go | 41 +----------- optimizer/optimizer.go | 2 +- optimizer/typeinferer.go | 114 +++++++++++++++++++++++++++++++--- optimizer/typeinferer_test.go | 88 ++++++++++++++++++++++++++ util/types/field_type.go | 44 +++++++++++++ 5 files changed, 243 insertions(+), 46 deletions(-) create mode 100644 optimizer/typeinferer_test.go diff --git a/ast/expressions.go b/ast/expressions.go index 3324368f5a..316a773d8d 100644 --- a/ast/expressions.go +++ b/ast/expressions.go @@ -14,13 +14,11 @@ package ast import ( - "fmt" "regexp" "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/parser/opcode" - "github.com/pingcap/tidb/util/charset" "github.com/pingcap/tidb/util/types" ) @@ -59,46 +57,13 @@ type ValueExpr struct { func NewValueExpr(value interface{}) *ValueExpr { ve := &ValueExpr{} ve.Data = types.RawData(value) - // TODO: make it more precise. - switch x := value.(type) { - case nil: - ve.Type = types.NewFieldType(mysql.TypeNull) - case bool, int64, int: - ve.Type = types.NewFieldType(mysql.TypeLonglong) - case uint64: - ve.Type = types.NewFieldType(mysql.TypeLonglong) - ve.Type.Flag |= mysql.UnsignedFlag - case string, UnquoteString: + if _, ok := value.(UnquoteString); ok { ve.Type = types.NewFieldType(mysql.TypeVarchar) ve.Type.Charset = mysql.DefaultCharset ve.Type.Collate = mysql.DefaultCollationName - case float64: - ve.Type = types.NewFieldType(mysql.TypeDouble) - case []byte: - ve.Type = types.NewFieldType(mysql.TypeBlob) - ve.Type.Charset = charset.CharsetBin - ve.Type.Collate = charset.CharsetBin - case mysql.Bit: - ve.Type = types.NewFieldType(mysql.TypeBit) - case mysql.Hex: - ve.Type = types.NewFieldType(mysql.TypeVarchar) - ve.Type.Charset = charset.CharsetBin - ve.Type.Collate = charset.CharsetBin - case mysql.Time: - ve.Type = types.NewFieldType(x.Type) - case mysql.Duration: - ve.Type = types.NewFieldType(mysql.TypeDuration) - case mysql.Decimal: - ve.Type = types.NewFieldType(mysql.TypeNewDecimal) - case mysql.Enum: - ve.Type = types.NewFieldType(mysql.TypeEnum) - case mysql.Set: - ve.Type = types.NewFieldType(mysql.TypeSet) - case *types.DataItem: - ve.Type = value.(*types.DataItem).Type - default: - panic(fmt.Sprintf("illegal literal value type:%T", value)) + return ve } + ve.Type = types.DefaultTypeForValue(value) return ve } diff --git a/optimizer/optimizer.go b/optimizer/optimizer.go index 089552c8dd..04a89a7584 100644 --- a/optimizer/optimizer.go +++ b/optimizer/optimizer.go @@ -34,7 +34,7 @@ func Optimize(is infoschema.InfoSchema, ctx context.Context, node ast.Node) (pla if err := ResolveName(node, is, ctx); err != nil { return nil, errors.Trace(err) } - if err := inferType(node); err != nil { + if err := InferType(node); err != nil { return nil, errors.Trace(err) } if err := preEvaluate(ctx, node); err != nil { diff --git a/optimizer/typeinferer.go b/optimizer/typeinferer.go index a6c09bfcc9..0da6c158dd 100644 --- a/optimizer/typeinferer.go +++ b/optimizer/typeinferer.go @@ -16,11 +16,14 @@ package optimizer import ( "github.com/pingcap/tidb/ast" "github.com/pingcap/tidb/context" + "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/optimizer/evaluator" + "github.com/pingcap/tidb/parser/opcode" + "github.com/pingcap/tidb/util/types" ) -// inferType infers result type for ast.ExprNode. -func inferType(node ast.Node) error { +// InferType infers result type for ast.ExprNode. +func InferType(node ast.Node) error { var inferrer typeInferrer node.Accept(&inferrer) return inferrer.err @@ -48,14 +51,111 @@ func (v *typeInferrer) Leave(in ast.Node) (out ast.Node, ok bool) { case *ast.FuncCastExpr: x.SetType(x.Tp) case *ast.SelectStmt: - rf := x.GetResultFields() - for _, val := range rf { - if val.Column.ID == 0 && val.Expr.GetType() != nil { - val.Column.FieldType = *(val.Expr.GetType()) + v.selectStmt(x) + case *ast.ParamMarkerExpr: + x.SetType(types.DefaultTypeForValue(x.GetValue())) + case *ast.BinaryOperationExpr: + v.binaryOperation(x) + case *ast.UnaryOperationExpr: + v.unaryOperation(x) + case *ast.BetweenExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.CompareSubqueryExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.ExistsSubqueryExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.PatternInExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.PatternLikeExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.PatternRegexpExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.IsNullExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.IsTruthExpr: + x.SetType(types.NewFieldType(mysql.TypeLonglong)) + case *ast.ParenthesesExpr: + x.SetType(x.Expr.GetType()) + // TODO: handle all expression types. + } + return in, true +} + +func (v *typeInferrer) selectStmt(x *ast.SelectStmt) { + rf := x.GetResultFields() + for _, val := range rf { + if val.Column.ID == 0 && val.Expr.GetType() != nil { + val.Column.FieldType = *(val.Expr.GetType()) + } + } +} + +func (v *typeInferrer) binaryOperation(x *ast.BinaryOperationExpr) { + switch x.Op { + case opcode.AndAnd, opcode.OrOr, opcode.LogicXor: + x.Type = types.NewFieldType(mysql.TypeLonglong) + case opcode.LT, opcode.LE, opcode.GE, opcode.GT, opcode.EQ, opcode.NE, opcode.NullEQ: + x.Type = types.NewFieldType(mysql.TypeLonglong) + case opcode.RightShift, opcode.LeftShift, opcode.And, opcode.Or, opcode.Xor: + x.Type = types.NewFieldType(mysql.TypeLonglong) + x.Type.Flag |= mysql.UnsignedFlag + case opcode.IntDiv: + x.Type = types.NewFieldType(mysql.TypeLonglong) + case opcode.Plus, opcode.Minus, opcode.Mul, opcode.Mod: + if x.L.GetType() != nil && x.R.GetType() != nil { + xTp := mergeArithType(x.L.GetType().Tp, x.R.GetType().Tp) + x.Type = types.NewFieldType(xTp) + leftUnsigned := x.L.GetType().Flag & mysql.UnsignedFlag + rightUnsigned := x.R.GetType().Flag & mysql.UnsignedFlag + // If both operand is unsigned, result is unsigned. + x.Type.Flag |= (leftUnsigned & rightUnsigned) + } + case opcode.Div: + if x.L.GetType() != nil && x.R.GetType() != nil { + xTp := mergeArithType(x.L.GetType().Tp, x.R.GetType().Tp) + if xTp == mysql.TypeLonglong { + xTp = mysql.TypeDecimal + } + x.Type = types.NewFieldType(xTp) + } + } +} + +func mergeArithType(a, b byte) byte { + switch a { + case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeDouble: + return mysql.TypeDouble + } + switch b { + case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeDouble, mysql.TypeFloat: + return mysql.TypeDouble + } + if a == mysql.TypeNewDecimal || b == mysql.TypeNewDecimal { + return mysql.TypeNewDecimal + } + return mysql.TypeLonglong +} + +func (v *typeInferrer) unaryOperation(x *ast.UnaryOperationExpr) { + switch x.Op { + case opcode.Not: + x.Type = types.NewFieldType(mysql.TypeLonglong) + case opcode.BitNeg: + x.Type = types.NewFieldType(mysql.TypeLonglong) + x.Type.Flag |= mysql.UnsignedFlag + case opcode.Plus: + x.Type = x.V.GetType() + case opcode.Minus: + x.Type = types.NewFieldType(mysql.TypeLonglong) + if x.V.GetType() != nil { + switch x.V.GetType().Tp { + case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeDouble, mysql.TypeFloat: + x.Type.Tp = mysql.TypeDouble + case mysql.TypeNewDecimal: + x.Type.Tp = mysql.TypeNewDecimal } } } - return in, true } type preEvaluator struct { diff --git a/optimizer/typeinferer_test.go b/optimizer/typeinferer_test.go new file mode 100644 index 0000000000..a6c7216d4e --- /dev/null +++ b/optimizer/typeinferer_test.go @@ -0,0 +1,88 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package optimizer_test + +import ( + . "github.com/pingcap/check" + "github.com/pingcap/tidb" + "github.com/pingcap/tidb/ast" + "github.com/pingcap/tidb/context" + "github.com/pingcap/tidb/mysql" + "github.com/pingcap/tidb/optimizer" + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/util/testkit" +) + +var _ = Suite(&testTypeInferrerSuite{}) + +type testTypeInferrerSuite struct { +} + +func (ts *testTypeInferrerSuite) TestInterType(c *C) { + store, err := tidb.NewStore(tidb.EngineGoLevelDBMemory) + c.Assert(err, IsNil) + defer store.Close() + testKit := testkit.NewTestKit(c, store) + testKit.MustExec("use test") + testKit.MustExec("create table t (c1 int, c2 double, c3 text)") + cases := []struct { + expr string + tp byte + }{ + {"c1", mysql.TypeLong}, + {"+1", mysql.TypeLonglong}, + {"-1", mysql.TypeLonglong}, + {"-'1'", mysql.TypeDouble}, + {"~1", mysql.TypeLonglong}, + {"!true", mysql.TypeLonglong}, + + {"c1 is true", mysql.TypeLonglong}, + {"c2 is null", mysql.TypeLonglong}, + {"cast(1 as decimal)", mysql.TypeNewDecimal}, + + {"1 and 1", mysql.TypeLonglong}, + {"1 or 1", mysql.TypeLonglong}, + {"1 xor 1", mysql.TypeLonglong}, + + {"'1' & 2", mysql.TypeLonglong}, + {"'1' | 2", mysql.TypeLonglong}, + {"'1' ^ 2", mysql.TypeLonglong}, + {"'1' << 1", mysql.TypeLonglong}, + {"'1' >> 1", mysql.TypeLonglong}, + + {"1 + '1'", mysql.TypeDouble}, + {"1 + 1.1", mysql.TypeNewDecimal}, + {"1 div 2", mysql.TypeLonglong}, + + {"1 > any (select 1)", mysql.TypeLonglong}, + {"exists (select 1)", mysql.TypeLonglong}, + {"1 in (2, 3)", mysql.TypeLonglong}, + {"'abc' like 'abc'", mysql.TypeLonglong}, + {"'abc' rlike 'abc'", mysql.TypeLonglong}, + {"(1+1)", mysql.TypeLonglong}, + } + for _, ca := range cases { + ctx := testKit.Se.(context.Context) + stmts, err := tidb.Parse(ctx, "select "+ca.expr+" from t") + c.Assert(err, IsNil) + c.Assert(len(stmts), Equals, 1) + stmt := stmts[0].(*ast.SelectStmt) + is := sessionctx.GetDomain(ctx).InfoSchema() + err = optimizer.ResolveName(stmt, is, ctx) + c.Assert(err, IsNil) + optimizer.InferType(stmt) + tp := stmt.GetResultFields()[0].Column.Tp + c.Assert(tp, Equals, ca.tp, Commentf("for %s", ca.expr)) + } +} diff --git a/util/types/field_type.go b/util/types/field_type.go index fb6531c3b7..53168bd351 100644 --- a/util/types/field_type.go +++ b/util/types/field_type.go @@ -142,3 +142,47 @@ func IsDataItem(d interface{}) bool { _, ok := d.(*DataItem) return ok } + +// DefaultTypeForValue returns the default FieldType for the value. +func DefaultTypeForValue(value interface{}) *FieldType { + switch x := value.(type) { + case nil: + return NewFieldType(mysql.TypeNull) + case bool, int64, int: + return NewFieldType(mysql.TypeLonglong) + case uint64: + tp := NewFieldType(mysql.TypeLonglong) + tp.Flag |= mysql.UnsignedFlag + return tp + case string: + tp := NewFieldType(mysql.TypeVarchar) + tp.Charset = mysql.DefaultCharset + tp.Collate = mysql.DefaultCollationName + return tp + case float64: + return NewFieldType(mysql.TypeNewDecimal) + case []byte: + tp := NewFieldType(mysql.TypeBlob) + tp.Charset = charset.CharsetBin + tp.Collate = charset.CharsetBin + case mysql.Bit: + return NewFieldType(mysql.TypeBit) + case mysql.Hex: + tp := NewFieldType(mysql.TypeVarchar) + tp.Charset = charset.CharsetBin + tp.Collate = charset.CharsetBin + case mysql.Time: + return NewFieldType(x.Type) + case mysql.Duration: + return NewFieldType(mysql.TypeDuration) + case mysql.Decimal: + return NewFieldType(mysql.TypeNewDecimal) + case mysql.Enum: + return NewFieldType(mysql.TypeEnum) + case mysql.Set: + return NewFieldType(mysql.TypeSet) + case *DataItem: + return value.(*DataItem).Type + } + return NewFieldType(mysql.TypeNull) +} From 2b58058501e51d7941b0cfa294fd294e7a54c0a2 Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Thu, 17 Dec 2015 10:20:20 +0800 Subject: [PATCH 02/63] optmizer: address comment --- optimizer/typeinferer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimizer/typeinferer.go b/optimizer/typeinferer.go index 0da6c158dd..e7424fb0cb 100644 --- a/optimizer/typeinferer.go +++ b/optimizer/typeinferer.go @@ -123,7 +123,7 @@ func (v *typeInferrer) binaryOperation(x *ast.BinaryOperationExpr) { func mergeArithType(a, b byte) byte { switch a { - case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeDouble: + case mysql.TypeString, mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeDouble, mysql.TypeFloat: return mysql.TypeDouble } switch b { From fbc938e7b690c89cb985468bd6f21c91da42eb93 Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Thu, 17 Dec 2015 14:56:11 +0800 Subject: [PATCH 03/63] optimizer: address comment --- optimizer/typeinferer.go | 2 +- util/types/field_type.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/optimizer/typeinferer.go b/optimizer/typeinferer.go index e7424fb0cb..41fca51c97 100644 --- a/optimizer/typeinferer.go +++ b/optimizer/typeinferer.go @@ -107,7 +107,7 @@ func (v *typeInferrer) binaryOperation(x *ast.BinaryOperationExpr) { x.Type = types.NewFieldType(xTp) leftUnsigned := x.L.GetType().Flag & mysql.UnsignedFlag rightUnsigned := x.R.GetType().Flag & mysql.UnsignedFlag - // If both operand is unsigned, result is unsigned. + // If both operands are unsigned, result is unsigned. x.Type.Flag |= (leftUnsigned & rightUnsigned) } case opcode.Div: diff --git a/util/types/field_type.go b/util/types/field_type.go index 53168bd351..64667b6231 100644 --- a/util/types/field_type.go +++ b/util/types/field_type.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + "github.com/ngaut/log" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/util/charset" ) @@ -182,7 +183,8 @@ func DefaultTypeForValue(value interface{}) *FieldType { case mysql.Set: return NewFieldType(mysql.TypeSet) case *DataItem: - return value.(*DataItem).Type + return x.Type } - return NewFieldType(mysql.TypeNull) + log.Errorf("Unknown value type %T for default field type.", value) + return nil } From 67ad875a29c1c75272f45539ce4d03b548920b4e Mon Sep 17 00:00:00 2001 From: shenli Date: Thu, 17 Dec 2015 23:56:16 +0800 Subject: [PATCH 04/63] *: Convert ErrKeyExists to MySQL error --- kv/index_iter.go | 5 ++-- kv/iter.go | 2 -- kv/union_store.go | 2 +- plan/plans/index.go | 4 +-- stmt/stmts/insert.go | 2 +- stmt/stmts/replace.go | 3 +-- table/tables/tables.go | 2 +- terror/terror.go | 55 ++++++++++++++++++++++++++++++++++++++ terror/terror_test.go | 8 ++++++ tidb-server/server/conn.go | 11 +++++--- 10 files changed, 80 insertions(+), 14 deletions(-) diff --git a/kv/index_iter.go b/kv/index_iter.go index 929682c71d..a25a8cf669 100644 --- a/kv/index_iter.go +++ b/kv/index_iter.go @@ -20,6 +20,7 @@ import ( "strings" "github.com/juju/errors" + "github.com/pingcap/tidb/terror" "github.com/pingcap/tidb/util/codec" ) @@ -169,7 +170,7 @@ func (c *kvIndex) Create(rm RetrieverMutator, indexedValues []interface{}, h int return errors.Trace(err) } - return errors.Trace(ErrKeyExists) + return errors.Trace(terror.ErrKeyExists) } // Delete removes the entry for handle h and indexdValues from KV index. @@ -258,7 +259,7 @@ func (c *kvIndex) Exist(rm RetrieverMutator, indexedValues []interface{}, h int6 } if handle != h { - return true, handle, errors.Trace(ErrKeyExists) + return true, handle, errors.Trace(terror.ErrKeyExists) } return true, handle, nil diff --git a/kv/iter.go b/kv/iter.go index b65d4e963e..faf8fa4a55 100644 --- a/kv/iter.go +++ b/kv/iter.go @@ -20,8 +20,6 @@ var ( ErrClosed = errors.New("Error: Transaction already closed") // ErrNotExist is used when try to get an entry with an unexist key from KV store. ErrNotExist = errors.New("Error: key not exist") - // ErrKeyExists is used when try to put an entry to KV store. - ErrKeyExists = errors.New("Error: key already exist") // ErrConditionNotMatch is used when condition is not met. ErrConditionNotMatch = errors.New("Error: Condition not match") // ErrLockConflict is used when try to lock an already locked key. diff --git a/kv/union_store.go b/kv/union_store.go index 7e11b13610..e5376aec6c 100644 --- a/kv/union_store.go +++ b/kv/union_store.go @@ -186,7 +186,7 @@ func (us *unionStore) CheckLazyConditionPairs() error { for ; it.Valid(); it.Next() { if len(it.Value()) == 0 { if _, exist := values[it.Key()]; exist { - return errors.Trace(ErrKeyExists) + return errors.Trace(terror.ErrKeyExists) } } else { if bytes.Compare(values[it.Key()], it.Value()) != 0 { diff --git a/plan/plans/index.go b/plan/plans/index.go index 3ea55f87cb..645efa44bf 100644 --- a/plan/plans/index.go +++ b/plan/plans/index.go @@ -436,12 +436,12 @@ func (r *indexPlan) pointLookup(ctx context.Context, val interface{}) (*plan.Row } var exist bool var h int64 - // We expect a kv.ErrKeyExists Error because we pass -1 as the handle which is not equal to the existed handle. + // We expect a terror.ErrKeyExists Error because we pass -1 as the handle which is not equal to the existed handle. exist, h, err = r.idx.Exist(txn, []interface{}{val}, -1) if !exist { return nil, errors.Trace(err) } - if terror.ErrorNotEqual(kv.ErrKeyExists, err) { + if terror.ErrorNotEqual(terror.ErrKeyExists, err) { return nil, errors.Trace(err) } var row *plan.Row diff --git a/stmt/stmts/insert.go b/stmt/stmts/insert.go index 7c298f1c06..831b990e5c 100644 --- a/stmt/stmts/insert.go +++ b/stmt/stmts/insert.go @@ -215,7 +215,7 @@ func (s *InsertIntoStmt) Exec(ctx context.Context) (_ rset.Recordset, err error) continue } - if len(s.OnDuplicate) == 0 || !terror.ErrorEqual(err, kv.ErrKeyExists) { + if len(s.OnDuplicate) == 0 || !terror.ErrorEqual(err, terror.ErrKeyExists) { return nil, errors.Trace(err) } if err = execOnDuplicateUpdate(ctx, t, row, h, toUpdateColumns); err != nil { diff --git a/stmt/stmts/replace.go b/stmt/stmts/replace.go index a6018f6771..07793a08d0 100644 --- a/stmt/stmts/replace.go +++ b/stmt/stmts/replace.go @@ -16,7 +16,6 @@ package stmts import ( "github.com/juju/errors" "github.com/pingcap/tidb/context" - "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/rset" "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/table" @@ -83,7 +82,7 @@ func (s *ReplaceIntoStmt) Exec(ctx context.Context) (_ rset.Recordset, err error if err == nil { continue } - if err != nil && !terror.ErrorEqual(err, kv.ErrKeyExists) { + if err != nil && !terror.ErrorEqual(err, terror.ErrKeyExists) { return nil, errors.Trace(err) } diff --git a/table/tables/tables.go b/table/tables/tables.go index 920636a845..267f65d82e 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -430,7 +430,7 @@ func (t *Table) AddRecord(ctx context.Context, r []interface{}, h int64) (record } colVals, _ := v.FetchValues(r) if err = v.X.Create(bs, colVals, recordID); err != nil { - if terror.ErrorEqual(err, kv.ErrKeyExists) { + if terror.ErrorEqual(err, terror.ErrKeyExists) { // Get the duplicate row handle // For insert on duplicate syntax, we should update the row iter, _, err1 := v.X.Seek(bs, colVals) diff --git a/terror/terror.go b/terror/terror.go index c4818d3b76..9b0a14ff74 100644 --- a/terror/terror.go +++ b/terror/terror.go @@ -19,6 +19,8 @@ import ( "strconv" "github.com/juju/errors" + "github.com/ngaut/log" + "github.com/pingcap/tidb/mysql" ) // Common base error instances. @@ -34,6 +36,8 @@ var ( UnknownSystemVar = ClassVariable.New(CodeUnknownSystemVar, "unknown system variable") MissConnectionID = ClassExpression.New(CodeMissConnectionID, "miss connection id information") + + ErrKeyExists = ClassKV.New(CodeKeyExists, "key already exist") ) // ErrCode represents a specific error type in a error class. @@ -57,6 +61,7 @@ const ( const ( CodeIncompatibleDBFormat ErrCode = iota + 1 CodeNoDataForHandle + CodeKeyExists ) // Variable error codes. @@ -190,6 +195,56 @@ func (e *Error) NotEqual(err error) bool { return !e.Equal(err) } +// ToSQLError convert Error to mysql.SQLError. +func (e *Error) ToSQLError() *mysql.SQLError { + code := e.getMySQLErrorCode() + return &mysql.SQLError{ + Code: code, + Message: e.message, + } +} + +var defaultMySQLErrorCode uint16 + +func (e *Error) getMySQLErrorCode() uint16 { + codeMap, ok := errClassToMySQLCodes[e.class] + if !ok { + log.Warnf("Unknown error class: %v", e.class) + return defaultMySQLErrorCode + } + code, ok := codeMap[e.code] + if !ok { + log.Warnf("Unknown error class: %v", e.class) + return defaultMySQLErrorCode + } + return code +} + +var parserMySQLErrCodes = map[ErrCode]uint16{} +var schemaMySQLErrCodes = map[ErrCode]uint16{} +var optimizerMySQLErrCodes = map[ErrCode]uint16{} +var executorMySQLErrCodes = map[ErrCode]uint16{} +var kvMySQLErrCodes = map[ErrCode]uint16{ + CodeKeyExists: mysql.ErrDupEntry, +} +var serverMySQLErrCodes = map[ErrCode]uint16{} +var expressionMySQLErrCodes = map[ErrCode]uint16{} + +var errClassToMySQLCodes map[ErrClass](map[ErrCode]uint16) + +func init() { + errClassToMySQLCodes = make(map[ErrClass](map[ErrCode]uint16)) + errClassToMySQLCodes[ClassParser] = parserMySQLErrCodes + errClassToMySQLCodes[ClassSchema] = schemaMySQLErrCodes + errClassToMySQLCodes[ClassOptimizer] = optimizerMySQLErrCodes + errClassToMySQLCodes[ClassExecutor] = executorMySQLErrCodes + errClassToMySQLCodes[ClassKV] = kvMySQLErrCodes + errClassToMySQLCodes[ClassServer] = serverMySQLErrCodes + errClassToMySQLCodes[ClassExpression] = expressionMySQLErrCodes + defaultMySQLErrorCode = mysql.ErrDupEntry + +} + // ErrorEqual returns a boolean indicating whether err1 is equal to err2. func ErrorEqual(err1, err2 error) bool { e1 := errors.Cause(err1) diff --git a/terror/terror_test.go b/terror/terror_test.go index 22ed3f63eb..30a1aa616e 100644 --- a/terror/terror_test.go +++ b/terror/terror_test.go @@ -19,6 +19,7 @@ import ( "github.com/juju/errors" . "github.com/pingcap/check" + "github.com/pingcap/tidb/mysql" ) func TestT(t *testing.T) { @@ -110,3 +111,10 @@ func (s *testTErrorSuite) TestErrorEqual(c *C) { c.Assert(ErrorEqual(te1, te3), IsFalse) c.Assert(ErrorEqual(te3, te4), IsFalse) } + +func (s *testTErrorSuite) TestMySQLErrorCode(c *C) { + ke := ErrKeyExists.Gen("key exists") + me := ke.ToSQLError() + c.Assert(me.Code, Equals, uint16(mysql.ErrDupEntry)) + c.Assert(me.Message, Equals, "key exists") +} diff --git a/tidb-server/server/conn.go b/tidb-server/server/conn.go index 8775920230..4b7e0503b0 100644 --- a/tidb-server/server/conn.go +++ b/tidb-server/server/conn.go @@ -319,10 +319,15 @@ func (cc *clientConn) writeOK() error { } func (cc *clientConn) writeError(e error) error { - var m *mysql.SQLError - var ok bool + var ( + m *mysql.SQLError + te *terror.Error + ok bool + ) originErr := errors.Cause(e) - if m, ok = originErr.(*mysql.SQLError); !ok { + if te, ok = originErr.(*terror.Error); ok { + m = te.ToSQLError() + } else { m = mysql.NewErrf(mysql.ErrUnknown, e.Error()) } From aad3d8097ffb4593e116fab9c9db61c91fec1967 Mon Sep 17 00:00:00 2001 From: shenli Date: Fri, 18 Dec 2015 11:14:09 +0800 Subject: [PATCH 05/63] *: Add test case --- terror/terror.go | 10 ++++++---- tidb-server/server/server_test.go | 23 ++++++++++++++++++++++- tidb-server/server/tidb_test.go | 4 ++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/terror/terror.go b/terror/terror.go index 9b0a14ff74..5fce102a56 100644 --- a/terror/terror.go +++ b/terror/terror.go @@ -198,10 +198,12 @@ func (e *Error) NotEqual(err error) bool { // ToSQLError convert Error to mysql.SQLError. func (e *Error) ToSQLError() *mysql.SQLError { code := e.getMySQLErrorCode() - return &mysql.SQLError{ - Code: code, - Message: e.message, - } + return mysql.NewErrf(code, e.message) + /* + return &mysql.SQLError{ + Code: code, + Message: e.message, + }*/ } var defaultMySQLErrorCode uint16 diff --git a/tidb-server/server/server_test.go b/tidb-server/server/server_test.go index 6b824d2046..1b12b78cb3 100644 --- a/tidb-server/server/server_test.go +++ b/tidb-server/server/server_test.go @@ -17,8 +17,9 @@ import ( "database/sql" "testing" - _ "github.com/go-sql-driver/mysql" + "github.com/go-sql-driver/mysql" . "github.com/pingcap/check" + tmysql "github.com/pingcap/tidb/mysql" ) func TestT(t *testing.T) { @@ -220,6 +221,26 @@ func runTestConcurrentUpdate(c *C) { }) } +func runTestErrorCode(c *C) { + runTests(c, dsn, func(dbt *DBTest) { + dbt.mustExec("create table test (c int PRIMARY KEY);") + dbt.mustExec("insert into test values (1);") + txn1, err := dbt.db.Begin() + c.Assert(err, IsNil) + _, err = txn1.Exec("insert into test values(1)") + c.Assert(err, IsNil) + err = txn1.Commit() + checkErrorCode(c, err, tmysql.ErrDupEntry) + }) +} + +func checkErrorCode(c *C, e error, code uint16) { + c.Assert(e, NotNil) + me, ok := e.(*mysql.MySQLError) + c.Assert(ok, IsTrue) + c.Assert(me.Number, Equals, code) +} + func runTestAuth(c *C) { runTests(c, dsn, func(dbt *DBTest) { dbt.mustExec(`CREATE USER 'test'@'%' IDENTIFIED BY '123';`) diff --git a/tidb-server/server/tidb_test.go b/tidb-server/server/tidb_test.go index 10579a7abc..e60ec6d025 100644 --- a/tidb-server/server/tidb_test.go +++ b/tidb-server/server/tidb_test.go @@ -68,6 +68,10 @@ func (ts *TidbTestSuite) TestConcurrentUpdate(c *C) { runTestConcurrentUpdate(c) } +func (ts *TidbTestSuite) TestErrorCode(c *C) { + runTestErrorCode(c) +} + func (ts *TidbTestSuite) TestAuth(c *C) { runTestAuth(c) } From 1871bb8acc64dd579418753fd7dee617c2c2d72f Mon Sep 17 00:00:00 2001 From: shenli Date: Fri, 18 Dec 2015 11:15:53 +0800 Subject: [PATCH 06/63] terror: Remove useless code --- terror/terror.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/terror/terror.go b/terror/terror.go index 5fce102a56..356f43a489 100644 --- a/terror/terror.go +++ b/terror/terror.go @@ -199,11 +199,6 @@ func (e *Error) NotEqual(err error) bool { func (e *Error) ToSQLError() *mysql.SQLError { code := e.getMySQLErrorCode() return mysql.NewErrf(code, e.message) - /* - return &mysql.SQLError{ - Code: code, - Message: e.message, - }*/ } var defaultMySQLErrorCode uint16 From a7402eba0dd3ed86788850a89a9ef22f2a16cc67 Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 12:30:39 +0800 Subject: [PATCH 07/63] kv: Clean up kv.go --- kv/key.go | 34 ++++++++++++++ kv/kv.go | 115 +--------------------------------------------- kv/union_store.go | 36 +++++++++++++++ kv/version.go | 38 +++++++++++++++ 4 files changed, 109 insertions(+), 114 deletions(-) create mode 100644 kv/key.go create mode 100644 kv/version.go diff --git a/kv/key.go b/kv/key.go new file mode 100644 index 0000000000..e92f25e7be --- /dev/null +++ b/kv/key.go @@ -0,0 +1,34 @@ +package kv + +import "bytes" + +// Key represents high-level Key type. +type Key []byte + +// Next returns the next key in byte-order. +func (k Key) Next() Key { + // add 0x0 to the end of key + buf := make([]byte, len([]byte(k))+1) + copy(buf, []byte(k)) + return buf +} + +// Cmp returns the comparison result of two key. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. +func (k Key) Cmp(another Key) int { + return bytes.Compare(k, another) +} + +// EncodedKey represents encoded key in low-level storage engine. +type EncodedKey []byte + +// Cmp returns the comparison result of two key. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. +func (k EncodedKey) Cmp(another EncodedKey) int { + return bytes.Compare(k, another) +} + +// Next returns the next key in byte-order. +func (k EncodedKey) Next() EncodedKey { + return EncodedKey(bytes.Join([][]byte{k, Key{0}}, nil)) +} diff --git a/kv/kv.go b/kv/kv.go index fc918338cb..c58695b2cf 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -13,98 +13,12 @@ package kv -import ( - "bytes" - "math" - - "github.com/juju/errors" -) - -// Key represents high-level Key type. -type Key []byte - -// Next returns the next key in byte-order. -func (k Key) Next() Key { - // add 0x0 to the end of key - buf := make([]byte, len([]byte(k))+1) - copy(buf, []byte(k)) - return buf -} - -// Cmp returns the comparison result of two key. -// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. -func (k Key) Cmp(another Key) int { - return bytes.Compare(k, another) -} - -// EncodedKey represents encoded key in low-level storage engine. -type EncodedKey []byte - -// Cmp returns the comparison result of two key. -// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. -func (k EncodedKey) Cmp(another EncodedKey) int { - return bytes.Compare(k, another) -} - -// Next returns the next key in byte-order. -func (k EncodedKey) Next() EncodedKey { - return EncodedKey(bytes.Join([][]byte{k, Key{0}}, nil)) -} - -// VersionProvider provides increasing IDs. -type VersionProvider interface { - CurrentVersion() (Version, error) -} - -// Version is the wrapper of KV's version. -type Version struct { - Ver uint64 -} - -var ( - // MaxVersion is the maximum version, notice that it's not a valid version. - MaxVersion = Version{Ver: math.MaxUint64} - // MinVersion is the minimum version, it's not a valid version, too. - MinVersion = Version{Ver: 0} -) - -// NewVersion creates a new Version struct. -func NewVersion(v uint64) Version { - return Version{ - Ver: v, - } -} - -// Cmp returns the comparison result of two versions. -// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. -func (v Version) Cmp(another Version) int { - if v.Ver > another.Ver { - return 1 - } else if v.Ver < another.Ver { - return -1 - } - return 0 -} - -// DecodeFn is a function that decodes data after fetching from store. -type DecodeFn func(raw interface{}) (interface{}, error) - -// EncodeFn is a function that encodes data before putting into store. -type EncodeFn func(raw interface{}) (interface{}, error) +import "github.com/juju/errors" // ErrNotCommitted is the error returned by CommitVersion when this // transaction is not committed. var ErrNotCommitted = errors.New("this transaction is not committed") -// Option is used for customizing kv store's behaviors during a transaction. -type Option int - -// Options is an interface of a set of options. Each option is associated with a value. -type Options interface { - // Get gets an option value. - Get(opt Option) (v interface{}, ok bool) -} - const ( // RangePrefetchOnCacheMiss directives that when dealing with a Get operation but failing to read data from cache, // it will launch a RangePrefetch to underlying storage instead of Get. The range starts from requested key and @@ -152,33 +66,6 @@ type MemBuffer interface { Release() } -// UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. -// Also, it provides some transaction related utilities. -type UnionStore interface { - MemBuffer - // Inc increases the value for key k in KV storage by step. - Inc(k Key, step int64) (int64, error) - // GetInt64 get int64 which created by Inc method. - GetInt64(k Key) (int64, error) - // CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched. - // Lazy condition pairs should be checked before transaction commit. - CheckLazyConditionPairs() error - // BatchPrefetch fetches values from KV storage to cache for later use. - BatchPrefetch(keys []Key) error - // RangePrefetch fetches values in the range [start, end] from KV storage - // to cache for later use. Maximum number of values is up to limit. - RangePrefetch(start, end Key, limit int) error - // WalkBuffer iterates all buffered kv pairs. - WalkBuffer(f func(k Key, v []byte) error) error - // SetOption sets an option with a value, when val is nil, uses the default - // value of this option. - SetOption(opt Option, val interface{}) - // DelOption deletes an option. - DelOption(opt Option) - // ReleaseSnapshot releases underlying snapshot. - ReleaseSnapshot() -} - // Transaction defines the interface for operations inside a Transaction. // This is not thread safe. type Transaction interface { diff --git a/kv/union_store.go b/kv/union_store.go index 7e11b13610..c896e62841 100644 --- a/kv/union_store.go +++ b/kv/union_store.go @@ -22,6 +22,42 @@ import ( "github.com/pingcap/tidb/terror" ) +// UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. +// Also, it provides some transaction related utilities. +type UnionStore interface { + MemBuffer + // Inc increases the value for key k in KV storage by step. + Inc(k Key, step int64) (int64, error) + // GetInt64 get int64 which created by Inc method. + GetInt64(k Key) (int64, error) + // CheckLazyConditionPairs loads all lazy values from store then checks if all values are matched. + // Lazy condition pairs should be checked before transaction commit. + CheckLazyConditionPairs() error + // BatchPrefetch fetches values from KV storage to cache for later use. + BatchPrefetch(keys []Key) error + // RangePrefetch fetches values in the range [start, end] from KV storage + // to cache for later use. Maximum number of values is up to limit. + RangePrefetch(start, end Key, limit int) error + // WalkBuffer iterates all buffered kv pairs. + WalkBuffer(f func(k Key, v []byte) error) error + // SetOption sets an option with a value, when val is nil, uses the default + // value of this option. + SetOption(opt Option, val interface{}) + // DelOption deletes an option. + DelOption(opt Option) + // ReleaseSnapshot releases underlying snapshot. + ReleaseSnapshot() +} + +// Option is used for customizing kv store's behaviors during a transaction. +type Option int + +// Options is an interface of a set of options. Each option is associated with a value. +type Options interface { + // Get gets an option value. + Get(opt Option) (v interface{}, ok bool) +} + var ( p = pool.NewCache("memdb pool", 100, func() interface{} { return NewMemDbBuffer() diff --git a/kv/version.go b/kv/version.go new file mode 100644 index 0000000000..42a14ed445 --- /dev/null +++ b/kv/version.go @@ -0,0 +1,38 @@ +package kv + +import "math" + +// VersionProvider provides increasing IDs. +type VersionProvider interface { + CurrentVersion() (Version, error) +} + +// Version is the wrapper of KV's version. +type Version struct { + Ver uint64 +} + +var ( + // MaxVersion is the maximum version, notice that it's not a valid version. + MaxVersion = Version{Ver: math.MaxUint64} + // MinVersion is the minimum version, it's not a valid version, too. + MinVersion = Version{Ver: 0} +) + +// NewVersion creates a new Version struct. +func NewVersion(v uint64) Version { + return Version{ + Ver: v, + } +} + +// Cmp returns the comparison result of two versions. +// The result will be 0 if a==b, -1 if a < b, and +1 if a > b. +func (v Version) Cmp(another Version) int { + if v.Ver > another.Ver { + return 1 + } else if v.Ver < another.Ver { + return -1 + } + return 0 +} From 34304b10507ff3ba17309df6be1dd71a040d8526 Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 12:36:57 +0800 Subject: [PATCH 08/63] Add licenses header --- kv/key.go | 13 +++++++++++++ kv/version.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/kv/key.go b/kv/key.go index e92f25e7be..d76f505fd3 100644 --- a/kv/key.go +++ b/kv/key.go @@ -1,3 +1,16 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package kv import "bytes" diff --git a/kv/version.go b/kv/version.go index 42a14ed445..f009215863 100644 --- a/kv/version.go +++ b/kv/version.go @@ -1,3 +1,16 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package kv import "math" From 9a943edeb933acedf3f50fe68309233426b1ddd6 Mon Sep 17 00:00:00 2001 From: shenli Date: Fri, 18 Dec 2015 12:51:36 +0800 Subject: [PATCH 09/63] terror: Address comment --- terror/terror.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/terror/terror.go b/terror/terror.go index 356f43a489..0208aea6d6 100644 --- a/terror/terror.go +++ b/terror/terror.go @@ -217,17 +217,21 @@ func (e *Error) getMySQLErrorCode() uint16 { return code } -var parserMySQLErrCodes = map[ErrCode]uint16{} -var schemaMySQLErrCodes = map[ErrCode]uint16{} -var optimizerMySQLErrCodes = map[ErrCode]uint16{} -var executorMySQLErrCodes = map[ErrCode]uint16{} -var kvMySQLErrCodes = map[ErrCode]uint16{ - CodeKeyExists: mysql.ErrDupEntry, -} -var serverMySQLErrCodes = map[ErrCode]uint16{} -var expressionMySQLErrCodes = map[ErrCode]uint16{} +var ( + // ErrCode to mysql error code map. + parserMySQLErrCodes = map[ErrCode]uint16{} + schemaMySQLErrCodes = map[ErrCode]uint16{} + optimizerMySQLErrCodes = map[ErrCode]uint16{} + executorMySQLErrCodes = map[ErrCode]uint16{} + kvMySQLErrCodes = map[ErrCode]uint16{ + CodeKeyExists: mysql.ErrDupEntry, + } + serverMySQLErrCodes = map[ErrCode]uint16{} + expressionMySQLErrCodes = map[ErrCode]uint16{} -var errClassToMySQLCodes map[ErrClass](map[ErrCode]uint16) + // ErrClass to code-map map. + errClassToMySQLCodes map[ErrClass](map[ErrCode]uint16) +) func init() { errClassToMySQLCodes = make(map[ErrClass](map[ErrCode]uint16)) @@ -238,8 +242,7 @@ func init() { errClassToMySQLCodes[ClassKV] = kvMySQLErrCodes errClassToMySQLCodes[ClassServer] = serverMySQLErrCodes errClassToMySQLCodes[ClassExpression] = expressionMySQLErrCodes - defaultMySQLErrorCode = mysql.ErrDupEntry - + defaultMySQLErrorCode = mysql.ErrUnknown } // ErrorEqual returns a boolean indicating whether err1 is equal to err2. From bbcdf33a91b4ce8cec5f2412e7c2b288e0e989bb Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 13:14:02 +0800 Subject: [PATCH 10/63] Remove compactor interface in kv.go --- kv/compactor.go | 40 ------------------------------ store/localstore/compactor.go | 20 +++++++++++---- store/localstore/compactor_test.go | 4 +-- 3 files changed, 17 insertions(+), 47 deletions(-) delete mode 100644 kv/compactor.go diff --git a/kv/compactor.go b/kv/compactor.go deleted file mode 100644 index 1687225e82..0000000000 --- a/kv/compactor.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2015 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package kv - -import "time" - -// CompactPolicy defines gc policy of MVCC storage. -type CompactPolicy struct { - // SafePoint specifies - SafePoint int - // TriggerInterval specifies how often should the compactor - // scans outdated data. - TriggerInterval time.Duration - // BatchDeleteCnt specifies the batch size for - // deleting outdated data transaction. - BatchDeleteCnt int -} - -// Compactor compacts MVCC storage. -type Compactor interface { - // OnGet is the hook point on Txn.Get. - OnGet(k Key) - // OnSet is the hook point on Txn.Set. - OnSet(k Key) - // OnDelete is the hook point on Txn.Delete. - OnDelete(k Key) - // Compact is the function removes the given key. - Compact(k Key) error -} diff --git a/store/localstore/compactor.go b/store/localstore/compactor.go index 2a70276261..97698fd831 100644 --- a/store/localstore/compactor.go +++ b/store/localstore/compactor.go @@ -25,13 +25,23 @@ import ( "github.com/pingcap/tidb/util/bytes" ) -var _ kv.Compactor = (*localstoreCompactor)(nil) - const ( deleteWorkerCnt = 3 ) -var localCompactDefaultPolicy = kv.CompactPolicy{ +// compactPolicy defines gc policy of MVCC storage. +type compactPolicy struct { + // SafePoint specifies + SafePoint int + // TriggerInterval specifies how often should the compactor + // scans outdated data. + TriggerInterval time.Duration + // BatchDeleteCnt specifies the batch size for + // deleting outdated data transaction. + BatchDeleteCnt int +} + +var localCompactDefaultPolicy = compactPolicy{ SafePoint: 20 * 1000, // in ms TriggerInterval: 10 * time.Second, BatchDeleteCnt: 100, @@ -45,7 +55,7 @@ type localstoreCompactor struct { workerWaitGroup *sync.WaitGroup ticker *time.Ticker db engine.DB - policy kv.CompactPolicy + policy compactPolicy } func (gc *localstoreCompactor) OnSet(k kv.Key) { @@ -196,7 +206,7 @@ func (gc *localstoreCompactor) Stop() { gc.workerWaitGroup.Wait() } -func newLocalCompactor(policy kv.CompactPolicy, db engine.DB) *localstoreCompactor { +func newLocalCompactor(policy compactPolicy, db engine.DB) *localstoreCompactor { return &localstoreCompactor{ recentKeys: make(map[string]struct{}), stopCh: make(chan struct{}), diff --git a/store/localstore/compactor_test.go b/store/localstore/compactor_test.go index 0f4da22d13..b3bf436b6f 100644 --- a/store/localstore/compactor_test.go +++ b/store/localstore/compactor_test.go @@ -46,7 +46,7 @@ func (s *localstoreCompactorTestSuite) TestCompactor(c *C) { db := store.(*dbStore).db store.(*dbStore).compactor.Stop() - policy := kv.CompactPolicy{ + policy := compactPolicy{ SafePoint: 500, BatchDeleteCnt: 1, TriggerInterval: 100 * time.Millisecond, @@ -119,7 +119,7 @@ func (s *localstoreCompactorTestSuite) TestStartStop(c *C) { db := store.(*dbStore).db for i := 0; i < 10000; i++ { - policy := kv.CompactPolicy{ + policy := compactPolicy{ SafePoint: 500, BatchDeleteCnt: 1, TriggerInterval: 100 * time.Millisecond, From 2321bc267ce45cc5f85b0497f2f205c67a6c920b Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 13:21:31 +0800 Subject: [PATCH 11/63] Group all errors together --- kv/iter.go | 22 ---------------------- kv/kv.go | 6 ------ kv/txn.go | 21 --------------------- 3 files changed, 49 deletions(-) diff --git a/kv/iter.go b/kv/iter.go index b65d4e963e..d2d4dd7dcd 100644 --- a/kv/iter.go +++ b/kv/iter.go @@ -15,28 +15,6 @@ package kv import "github.com/juju/errors" -var ( - // ErrClosed is used when close an already closed txn. - ErrClosed = errors.New("Error: Transaction already closed") - // ErrNotExist is used when try to get an entry with an unexist key from KV store. - ErrNotExist = errors.New("Error: key not exist") - // ErrKeyExists is used when try to put an entry to KV store. - ErrKeyExists = errors.New("Error: key already exist") - // ErrConditionNotMatch is used when condition is not met. - ErrConditionNotMatch = errors.New("Error: Condition not match") - // ErrLockConflict is used when try to lock an already locked key. - ErrLockConflict = errors.New("Error: Lock conflict") - // ErrLazyConditionPairsNotMatch is used when value in store differs from expect pairs. - ErrLazyConditionPairsNotMatch = errors.New("Error: Lazy condition pairs not match") - // ErrRetryable is used when KV store occurs RPC error or some other - // errors which SQL layer can safely retry. - ErrRetryable = errors.New("Error: KV error safe to retry") - // ErrCannotSetNilValue is the error when sets an empty value. - ErrCannotSetNilValue = errors.New("can not set nil value") - // ErrInvalidTxn is the error when commits or rollbacks in an invalid transaction. - ErrInvalidTxn = errors.New("invalid transaction") -) - // NextUntil applies FnKeyCmp to each entry of the iterator until meets some condition. // It will stop when fn returns true, or iterator is invalid or an error occurs. func NextUntil(it Iterator, fn FnKeyCmp) error { diff --git a/kv/kv.go b/kv/kv.go index c58695b2cf..fab9416c82 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -13,12 +13,6 @@ package kv -import "github.com/juju/errors" - -// ErrNotCommitted is the error returned by CommitVersion when this -// transaction is not committed. -var ErrNotCommitted = errors.New("this transaction is not committed") - const ( // RangePrefetchOnCacheMiss directives that when dealing with a Get operation but failing to read data from cache, // it will launch a RangePrefetch to underlying storage instead of Get. The range starts from requested key and diff --git a/kv/txn.go b/kv/txn.go index 2bf483970d..197a6fc5c5 100644 --- a/kv/txn.go +++ b/kv/txn.go @@ -16,33 +16,12 @@ package kv import ( "math" "math/rand" - "strings" "time" "github.com/juju/errors" "github.com/ngaut/log" - "github.com/pingcap/go-themis" - "github.com/pingcap/tidb/terror" ) -// IsRetryableError checks if the err is a fatal error and the under going operation is worth to retry. -func IsRetryableError(err error) bool { - if err == nil { - return false - } - - if terror.ErrorEqual(err, ErrRetryable) || - terror.ErrorEqual(err, ErrLockConflict) || - terror.ErrorEqual(err, ErrConditionNotMatch) || - terror.ErrorEqual(err, themis.ErrRetryable) || - // HBase exception message will tell you if you should retry or not - strings.Contains(err.Error(), "try again later") { - return true - } - - return false -} - // RunInNewTxn will run the f in a new transaction environment. func RunInNewTxn(store Storage, retryable bool, f func(txn Transaction) error) error { for i := 0; i < maxRetryCnt; i++ { From 32684c03256e05e37758f3c0cf132a1e40469ae7 Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 13:21:52 +0800 Subject: [PATCH 12/63] Add error.go to kv --- kv/error.go | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 kv/error.go diff --git a/kv/error.go b/kv/error.go new file mode 100644 index 0000000000..ef5a3c095f --- /dev/null +++ b/kv/error.go @@ -0,0 +1,66 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kv + +import ( + "errors" + "strings" + + "github.com/pingcap/go-themis" + "github.com/pingcap/tidb/terror" +) + +var ( + // ErrClosed is used when close an already closed txn. + ErrClosed = errors.New("Error: Transaction already closed") + // ErrNotExist is used when try to get an entry with an unexist key from KV store. + ErrNotExist = errors.New("Error: key not exist") + // ErrKeyExists is used when try to put an entry to KV store. + ErrKeyExists = errors.New("Error: key already exist") + // ErrConditionNotMatch is used when condition is not met. + ErrConditionNotMatch = errors.New("Error: Condition not match") + // ErrLockConflict is used when try to lock an already locked key. + ErrLockConflict = errors.New("Error: Lock conflict") + // ErrLazyConditionPairsNotMatch is used when value in store differs from expect pairs. + ErrLazyConditionPairsNotMatch = errors.New("Error: Lazy condition pairs not match") + // ErrRetryable is used when KV store occurs RPC error or some other + // errors which SQL layer can safely retry. + ErrRetryable = errors.New("Error: KV error safe to retry") + // ErrCannotSetNilValue is the error when sets an empty value. + ErrCannotSetNilValue = errors.New("can not set nil value") + // ErrInvalidTxn is the error when commits or rollbacks in an invalid transaction. + ErrInvalidTxn = errors.New("invalid transaction") + + // ErrNotCommitted is the error returned by CommitVersion when this + // transaction is not committed. + ErrNotCommitted = errors.New("this transaction has not committed") +) + +// IsRetryableError checks if the err is a fatal error and the under going operation is worth to retry. +func IsRetryableError(err error) bool { + if err == nil { + return false + } + + if terror.ErrorEqual(err, ErrRetryable) || + terror.ErrorEqual(err, ErrLockConflict) || + terror.ErrorEqual(err, ErrConditionNotMatch) || + terror.ErrorEqual(err, themis.ErrRetryable) || + // HBase exception message will tell you if you should retry or not + strings.Contains(err.Error(), "try again later") { + return true + } + + return false +} From e331d3543af1e472925ffde88b03d7d25cdbeb72 Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 13:40:25 +0800 Subject: [PATCH 13/63] Address comments --- kv/error.go | 9 +++++++++ kv/union_store.go | 10 ---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/kv/error.go b/kv/error.go index ef5a3c095f..23523a1e16 100644 --- a/kv/error.go +++ b/kv/error.go @@ -64,3 +64,12 @@ func IsRetryableError(err error) bool { return false } + +// IsErrNotFound checks if err is a kind of NotFound error. +func IsErrNotFound(err error) bool { + if terror.ErrorEqual(err, ErrNotExist) { + return true + } + + return false +} diff --git a/kv/union_store.go b/kv/union_store.go index c896e62841..d286e5786c 100644 --- a/kv/union_store.go +++ b/kv/union_store.go @@ -19,7 +19,6 @@ import ( "github.com/juju/errors" "github.com/ngaut/pool" - "github.com/pingcap/tidb/terror" ) // UnionStore is a store that wraps a snapshot for read and a BufferStore for buffered write. @@ -64,15 +63,6 @@ var ( }) ) -// IsErrNotFound checks if err is a kind of NotFound error. -func IsErrNotFound(err error) bool { - if terror.ErrorEqual(err, ErrNotExist) { - return true - } - - return false -} - // UnionStore is an in-memory Store which contains a buffer for write and a // snapshot for read. type unionStore struct { From 108dc596e3f6f28a5c7f66329e0c02a90cfd1e24 Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Fri, 18 Dec 2015 13:49:45 +0800 Subject: [PATCH 14/63] optimizer: address comment. --- optimizer/typeinferer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optimizer/typeinferer.go b/optimizer/typeinferer.go index 41fca51c97..6b3166fd62 100644 --- a/optimizer/typeinferer.go +++ b/optimizer/typeinferer.go @@ -84,6 +84,8 @@ func (v *typeInferrer) Leave(in ast.Node) (out ast.Node, ok bool) { func (v *typeInferrer) selectStmt(x *ast.SelectStmt) { rf := x.GetResultFields() for _, val := range rf { + // column ID is 0 means it is not a real column from table, but a temporary column, + // so its type is not pre-defined, we need to set it. if val.Column.ID == 0 && val.Expr.GetType() != nil { val.Column.FieldType = *(val.Expr.GetType()) } From c7f33cd14b0e1158a37635378c3011619a962c8d Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 13:50:31 +0800 Subject: [PATCH 15/63] kv: clean up more --- kv/index_iter.go | 24 ++++++++++++++++++++++++ kv/kv.go | 24 ------------------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/kv/index_iter.go b/kv/index_iter.go index 929682c71d..303dc288b1 100644 --- a/kv/index_iter.go +++ b/kv/index_iter.go @@ -28,6 +28,30 @@ var ( _ IndexIterator = (*indexIter)(nil) ) +// IndexIterator is the interface for iterator of index data on KV store. +type IndexIterator interface { + Next() (k []interface{}, h int64, err error) + Close() +} + +// Index is the interface for index data on KV store. +type Index interface { + // Create supports insert into statement. + Create(rm RetrieverMutator, indexedValues []interface{}, h int64) error + // Delete supports delete from statement. + Delete(m Mutator, indexedValues []interface{}, h int64) error + // Drop supports drop table, drop index statements. + Drop(rm RetrieverMutator) error + // Exist supports check index exists or not. + Exist(rm RetrieverMutator, indexedValues []interface{}, h int64) (bool, int64, error) + // GenIndexKey generates an index key. + GenIndexKey(indexedValues []interface{}, h int64) (key []byte, distinct bool, err error) + // Seek supports where clause. + Seek(r Retriever, indexedValues []interface{}) (iter IndexIterator, hit bool, err error) + // SeekFirst supports aggregate min and ascend order by. + SeekFirst(r Retriever) (iter IndexIterator, err error) +} + func encodeHandle(h int64) []byte { buf := &bytes.Buffer{} err := binary.Write(buf, binary.BigEndian, h) diff --git a/kv/kv.go b/kv/kv.go index fab9416c82..68369bf1eb 100644 --- a/kv/kv.go +++ b/kv/kv.go @@ -123,27 +123,3 @@ type Iterator interface { Valid() bool Close() } - -// IndexIterator is the interface for iterator of index data on KV store. -type IndexIterator interface { - Next() (k []interface{}, h int64, err error) - Close() -} - -// Index is the interface for index data on KV store. -type Index interface { - // Create supports insert into statement. - Create(rm RetrieverMutator, indexedValues []interface{}, h int64) error - // Delete supports delete from statement. - Delete(m Mutator, indexedValues []interface{}, h int64) error - // Drop supports drop table, drop index statements. - Drop(rm RetrieverMutator) error - // Exist supports check index exists or not. - Exist(rm RetrieverMutator, indexedValues []interface{}, h int64) (bool, int64, error) - // GenIndexKey generates an index key. - GenIndexKey(indexedValues []interface{}, h int64) (key []byte, distinct bool, err error) - // Seek supports where clause. - Seek(r Retriever, indexedValues []interface{}) (iter IndexIterator, hit bool, err error) - // SeekFirst supports aggregate min and ascend order by. - SeekFirst(r Retriever) (iter IndexIterator, err error) -} From d0999dcf49022ae46afa3f0ba545a91a9b66dc80 Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Fri, 18 Dec 2015 14:37:17 +0800 Subject: [PATCH 16/63] optimizer: use HasLen checker. --- optimizer/typeinferer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimizer/typeinferer_test.go b/optimizer/typeinferer_test.go index a6c7216d4e..037669fd7d 100644 --- a/optimizer/typeinferer_test.go +++ b/optimizer/typeinferer_test.go @@ -76,7 +76,7 @@ func (ts *testTypeInferrerSuite) TestInterType(c *C) { ctx := testKit.Se.(context.Context) stmts, err := tidb.Parse(ctx, "select "+ca.expr+" from t") c.Assert(err, IsNil) - c.Assert(len(stmts), Equals, 1) + c.Assert(stmts, HasLen, 1) stmt := stmts[0].(*ast.SelectStmt) is := sessionctx.GetDomain(ctx).InfoSchema() err = optimizer.ResolveName(stmt, is, ctx) From 1f9964cab68fa6d28290c2545b4763927e6b9c0a Mon Sep 17 00:00:00 2001 From: ngaut Date: Fri, 18 Dec 2015 21:21:22 +0800 Subject: [PATCH 17/63] store: Don't do any seek operation after store closed. --- store/localstore/kv.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/store/localstore/kv.go b/store/localstore/kv.go index e6e77683fe..8712acb8c9 100644 --- a/store/localstore/kv.go +++ b/store/localstore/kv.go @@ -101,7 +101,8 @@ func (s *dbStore) CommitTxn(txn *dbTxn) error { return errors.Trace(err) } -func (s *dbStore) seekWorker(seekCh chan *command) { +func (s *dbStore) seekWorker(wg *sync.WaitGroup, seekCh chan *command) { + defer wg.Done() for { var pending []*command select { @@ -131,8 +132,10 @@ func (s *dbStore) seekWorker(seekCh chan *command) { func (s *dbStore) scheduler() { closed := false seekCh := make(chan *command, 1000) + wgSeekWorkers := &sync.WaitGroup{} + wgSeekWorkers.Add(maxSeekWorkers) for i := 0; i < maxSeekWorkers; i++ { - go s.seekWorker(seekCh) + go s.seekWorker(wgSeekWorkers, seekCh) } for { @@ -150,9 +153,10 @@ func (s *dbStore) scheduler() { } case <-s.closeCh: closed = true - s.wg.Done() // notify seek worker to exit close(seekCh) + wgSeekWorkers.Wait() + s.wg.Done() } } } From f71e4ffbbd8c01266434768ac039ad380352d9df Mon Sep 17 00:00:00 2001 From: shenli Date: Sat, 19 Dec 2015 09:48:36 +0800 Subject: [PATCH 18/63] *: Address comment --- tidb-server/server/server_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tidb-server/server/server_test.go b/tidb-server/server/server_test.go index 1b12b78cb3..ab2a95a4eb 100644 --- a/tidb-server/server/server_test.go +++ b/tidb-server/server/server_test.go @@ -235,7 +235,6 @@ func runTestErrorCode(c *C) { } func checkErrorCode(c *C, e error, code uint16) { - c.Assert(e, NotNil) me, ok := e.(*mysql.MySQLError) c.Assert(ok, IsTrue) c.Assert(me.Number, Equals, code) From 1b0c2e38cfbddb37d6714d3b477b8ca9d848cd3a Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sat, 19 Dec 2015 22:29:13 +0800 Subject: [PATCH 19/63] builtin/time: support CURRENT_TIME(), CURTIME() For github issue #236. --- expression/builtin/builtin.go | 2 ++ expression/builtin/time.go | 7 +++++++ mysql/time.go | 9 +++++++-- parser/parser.y | 10 ++++++++++ parser/scanner.l | 6 ++++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/expression/builtin/builtin.go b/expression/builtin/builtin.go index d5c73d5b4e..0d266f3d4b 100644 --- a/expression/builtin/builtin.go +++ b/expression/builtin/builtin.go @@ -69,7 +69,9 @@ var Funcs = map[string]Func{ // time functions "curdate": {builtinCurrentDate, 0, 0, false, false}, "current_date": {builtinCurrentDate, 0, 0, false, false}, + "current_time": {builtinCurrentTime, 0, 0, false, false}, "current_timestamp": {builtinNow, 0, 1, false, false}, + "curtime": {builtinCurrentTime, 0, 0, false, false}, "date": {builtinDate, 8, 8, true, false}, "day": {builtinDay, 1, 1, true, false}, "dayofmonth": {builtinDayOfMonth, 1, 1, true, false}, diff --git a/expression/builtin/time.go b/expression/builtin/time.go index ac4204e61d..3c0f9b169d 100644 --- a/expression/builtin/time.go +++ b/expression/builtin/time.go @@ -309,6 +309,13 @@ func builtinCurrentDate(args []interface{}, ctx map[interface{}]interface{}) (in Type: mysql.TypeDate, Fsp: 0}, nil } +// See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_curtime +func builtinCurrentTime(args []interface{}, ctx map[interface{}]interface{}) (interface{}, error) { + return mysql.Time{ + Time: time.Now(), + Type: mysql.TypeDuration, Fsp: 0}, nil +} + func checkFsp(arg interface{}) (int, error) { fsp, err := types.ToInt64(arg) if err != nil { diff --git a/mysql/time.go b/mysql/time.go index 6eb2c1f4cd..446b2aaa19 100644 --- a/mysql/time.go +++ b/mysql/time.go @@ -34,8 +34,9 @@ var ( // Time format without fractional seconds precision. const ( - DateFormat = "2006-01-02" - TimeFormat = "2006-01-02 15:04:05" + DateFormat = "2006-01-02" + CurtimeFormat = "15:04:05" + TimeFormat = "2006-01-02 15:04:05" // TimeFSPFormat is time format with fractional seconds precision. TimeFSPFormat = "2006-01-02 15:04:05.000000" ) @@ -125,6 +126,10 @@ func (t Time) String() string { return t.Time.Format(DateFormat) } + if t.Type == TypeDuration { + return t.Time.Format(CurtimeFormat) + } + tfStr := TimeFormat if t.Fsp > 0 { tfStr = fmt.Sprintf("%s.%s", tfStr, strings.Repeat("0", t.Fsp)) diff --git a/parser/parser.y b/parser/parser.y index b421325596..9c5d670d84 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -104,6 +104,8 @@ import ( cross "CROSS" curDate "CURDATE" currentDate "CURRENT_DATE" + curTime "CURTIME" + currentTime "CURRENT_TIME" currentUser "CURRENT_USER" database "DATABASE" databases "DATABASES" @@ -2137,6 +2139,14 @@ FunctionCallNonKeyword: { $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} } +| "CURTIME" '(' ')' + { + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} + } +| "CURRENT_TIME" '(' ')' + { + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} + } | "CURRENT_TIMESTAMP" FuncDatetimePrec { args := []ast.ExprNode{} diff --git a/parser/scanner.l b/parser/scanner.l index af5d5d6acd..9fdfd1fc6c 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -323,6 +323,8 @@ create {c}{r}{e}{a}{t}{e} cross {c}{r}{o}{s}{s} curdate {c}{u}{r}{d}{a}{t}{e} current_date {c}{u}{r}{r}{e}{n}{t}_{d}{a}{t}{e} +curtime {c}{u}{r}{t}{i}{m}{e} +current_time {c}{u}{r}{r}{e}{n}{t}_{t}{i}{m}{e} current_user {c}{u}{r}{r}{e}{n}{t}_{u}{s}{e}{r} database {d}{a}{t}{a}{b}{a}{s}{e} databases {d}{a}{t}{a}{b}{a}{s}{e}{s} @@ -692,6 +694,10 @@ year_month {y}{e}{a}{r}_{m}{o}{n}{t}{h} return curDate {current_date} lval.item = string(l.val) return currentDate +{curtime} lval.item = string(l.val) + return curTime +{current_time} lval.item = string(l.val) + return currentTime {current_user} lval.item = string(l.val) return currentUser {database} lval.item = string(l.val) From c01879ba2e334fb40fb3aee48a67148f263e2d2d Mon Sep 17 00:00:00 2001 From: sllt Date: Sat, 19 Dec 2015 23:51:04 +0800 Subject: [PATCH 20/63] parser: support built-in function pow --- expression/builtin/builtin.go | 6 ++++-- expression/builtin/math.go | 19 +++++++++++++++++ expression/builtin/math_test.go | 36 +++++++++++++++++++++++++++++++++ parser/parser.y | 12 +++++++++++ parser/scanner.l | 6 ++++++ 5 files changed, 77 insertions(+), 2 deletions(-) diff --git a/expression/builtin/builtin.go b/expression/builtin/builtin.go index d5c73d5b4e..15c20232fa 100644 --- a/expression/builtin/builtin.go +++ b/expression/builtin/builtin.go @@ -55,8 +55,10 @@ var Funcs = map[string]Func{ "coalesce": {builtinCoalesce, 1, -1, true, false}, // math functions - "abs": {builtinAbs, 1, 1, true, false}, - "rand": {builtinRand, 0, 1, true, false}, + "abs": {builtinAbs, 1, 1, true, false}, + "rand": {builtinRand, 0, 1, true, false}, + "pow": {builtinPow, 2, 2, true, false}, + "power": {builtinPower, 2, 2, true, false}, // group by functions "avg": {builtinAvg, 1, 1, false, true}, diff --git a/expression/builtin/math.go b/expression/builtin/math.go index d3cce6fe41..dc2b04971b 100644 --- a/expression/builtin/math.go +++ b/expression/builtin/math.go @@ -63,3 +63,22 @@ func builtinRand(args []interface{}, ctx map[interface{}]interface{}) (v interfa return rand.Float64(), nil } + +func builtinPow(args []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + x, err := types.ToFloat64(args[0]) + if err != nil { + return nil, errors.Trace(err) + } + + y, err := types.ToFloat64(args[1]) + if err != nil { + return nil, errors.Trace(err) + } + + return math.Pow(x, y), nil + +} + +func builtinPower(args []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { + return builtinPow(args, ctx) +} diff --git a/expression/builtin/math_test.go b/expression/builtin/math_test.go index 255238e70a..a89d07fdb5 100644 --- a/expression/builtin/math_test.go +++ b/expression/builtin/math_test.go @@ -43,3 +43,39 @@ func (s *testBuiltinSuite) TestRand(c *C) { c.Assert(v, Less, float64(1)) c.Assert(v, GreaterEqual, float64(0)) } + +func (s *testBuiltinSuite) TestPow(c *C) { + tbl := []struct { + Arg []interface{} + Ret float64 + }{ + {[]interface{}{1, 3}, 1}, + {[]interface{}{2, 2}, 4}, + {[]interface{}{4, 0.5}, 2}, + {[]interface{}{4, -2}, 0.0625}, + } + + for _, t := range tbl { + v, err := builtinPow(t.Arg, nil) + c.Assert(err, IsNil) + c.Assert(v, DeepEquals, t.Ret) + } +} + +func (s *testBuiltinSuite) TestPower(c *C) { + tbl := []struct { + Arg []interface{} + Ret float64 + }{ + {[]interface{}{1, 3}, 1}, + {[]interface{}{2, 2}, 4}, + {[]interface{}{4, 0.5}, 2}, + {[]interface{}{4, -2}, 0.0625}, + } + + for _, t := range tbl { + v, err := builtinPower(t.Arg, nil) + c.Assert(err, IsNil) + c.Assert(v, DeepEquals, t.Ret) + } +} diff --git a/parser/parser.y b/parser/parser.y index b421325596..c797b0bb28 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -210,6 +210,8 @@ import ( outer "OUTER" password "PASSWORD" placeholder "PLACEHOLDER" + pow "POW" + power "POWER" prepare "PREPARE" primary "PRIMARY" quarter "QUARTER" @@ -2257,6 +2259,16 @@ FunctionCallNonKeyword: { $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: $3.([]ast.ExprNode)} } +| "POW" '(' Expression ',' Expression ')' + { + args := []ast.ExprNode{$3.(ast.ExprNode), $5.(ast.ExprNode)} + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: args} + } +| "POWER" '(' Expression ',' Expression ')' + { + args := []ast.ExprNode{$3.(ast.ExprNode), $5.(ast.ExprNode)} + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: args} + } | "RAND" '(' ExpressionOpt ')' { diff --git a/parser/scanner.l b/parser/scanner.l index af5d5d6acd..0433f776e5 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -414,6 +414,8 @@ or {o}{r} order {o}{r}{d}{e}{r} outer {o}{u}{t}{e}{r} password {p}{a}{s}{s}{w}{o}{r}{d} +pow {p}{o}{w} +power {p}{o}{w}{e}{r} prepare {p}{r}{e}{p}{a}{r}{e} primary {p}{r}{i}{m}{a}{r}{y} quarter {q}{u}{a}{r}{t}{e}{r} @@ -850,6 +852,10 @@ year_month {y}{e}{a}{r}_{m}{o}{n}{t}{h} {outer} return outer {password} lval.item = string(l.val) return password +{pow} lval.item = string(l.val) + return pow +{power} lval.item = string(l.val) + return power {prepare} lval.item = string(l.val) return prepare {primary} return primary From 9908dd49b2b7e5ee27b0e729d7234a537c9bbf49 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 00:15:23 +0800 Subject: [PATCH 21/63] doc: fix some typos and syntax errors. --- README.md | 6 +++--- docs/QUICKSTART.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9ac43bc89a..88c4c3f782 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Grow TiDB as your business grows. You can increase the capacity simply by adding Evolve TiDB schemas as your requirement evolves. You can add new columns and indices without stopping or affecting the on-going operations. - __Consistent distributed transactions__ -Think TiDB as a single-machine RDBMS. You can start a transaction that acrosses multiple machines without worrying about consistency. TiDB makes your application code simple and robust. +Think TiDB as a single-machine RDBMS. You can start a transaction that crosses multiple machines without worrying about consistency. TiDB makes your application code simple and robust. - __Compatible with MySQL protocol__ Use TiDB as MySQL. You can replace MySQL with TiDB to power your application without changing a single line of code in most cases. @@ -22,10 +22,10 @@ Enjoy TiDB as much as we love Go. We believe Go code is both easy and enjoyable - __NewSQL over HBase__ -Turns HBase into NewSQL database +Turn HBase into NewSQL database - __Multiple storage engine support__ -Power TiDB with your most favorite engines. TiDB supports many popular storage engines in single-machine mode. You can choose from goleveldb, LevelDB, RocksDB, LMDB, BoltDB and even more to come. +Power TiDB with your most favorite engines. TiDB supports many popular storage engines in single-machine mode. You can choose from GolevelDB, LevelDB, RocksDB, LMDB, BoltDB and even more to come. ## Status diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index faea863103..89c82717ae 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -2,10 +2,10 @@ #### Run TiDB with docker -You can quickly test tidb with docker, the source repository contains the Dockerfile which +You can quickly test TiDB with docker, the source repository contains the Dockerfile which contains local tidb-server. -Or you can pull TiDB docker image contains HBase standalone and then run TiDB as distributed database in a docker container. +Or you can pull TiDB docker image contains HBase standalone and then run TiDB as a distributed database in a docker container. To install docker on your system, you can read the document on https://docs.docker.com/ From 8a0a7544bdd4344eaa549cf30f024c5809793cdb Mon Sep 17 00:00:00 2001 From: sllt Date: Sun, 20 Dec 2015 00:52:20 +0800 Subject: [PATCH 22/63] parser: support built-in function pow --- expression/builtin/builtin.go | 2 +- expression/builtin/math_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/expression/builtin/builtin.go b/expression/builtin/builtin.go index 15c20232fa..6f80b02d69 100644 --- a/expression/builtin/builtin.go +++ b/expression/builtin/builtin.go @@ -56,9 +56,9 @@ var Funcs = map[string]Func{ // math functions "abs": {builtinAbs, 1, 1, true, false}, - "rand": {builtinRand, 0, 1, true, false}, "pow": {builtinPow, 2, 2, true, false}, "power": {builtinPower, 2, 2, true, false}, + "rand": {builtinRand, 0, 1, true, false}, // group by functions "avg": {builtinAvg, 1, 1, false, true}, diff --git a/expression/builtin/math_test.go b/expression/builtin/math_test.go index a89d07fdb5..96621da301 100644 --- a/expression/builtin/math_test.go +++ b/expression/builtin/math_test.go @@ -60,6 +60,21 @@ func (s *testBuiltinSuite) TestPow(c *C) { c.Assert(err, IsNil) c.Assert(v, DeepEquals, t.Ret) } + + t := []interface{}{'a', 3} + _, err := builtinPow(t, nil) + c.Assert(err, NotNil) + t = []interface{}{3, 'a'} + _, err = builtinPow(t, nil) + c.Assert(err, NotNil) + + t = []interface{}{nil, 3} + _, err = builtinPow(t, nil) + c.Assert(err, NotNil) + t = []interface{}{3, nil} + _, err = builtinPow(t, nil) + c.Assert(err, NotNil) + } func (s *testBuiltinSuite) TestPower(c *C) { @@ -78,4 +93,19 @@ func (s *testBuiltinSuite) TestPower(c *C) { c.Assert(err, IsNil) c.Assert(v, DeepEquals, t.Ret) } + + t := []interface{}{'a', 3} + _, err := builtinPower(t, nil) + c.Assert(err, NotNil) + t = []interface{}{3, 'a'} + _, err = builtinPower(t, nil) + c.Assert(err, NotNil) + + t = []interface{}{nil, 3} + _, err = builtinPower(t, nil) + c.Assert(err, NotNil) + t = []interface{}{3, nil} + _, err = builtinPower(t, nil) + c.Assert(err, NotNil) + } From ee333b554ef7343cb6bb49898fb2e90854f57a63 Mon Sep 17 00:00:00 2001 From: sllt Date: Sun, 20 Dec 2015 01:06:36 +0800 Subject: [PATCH 23/63] fix alignment --- parser/scanner.l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/scanner.l b/parser/scanner.l index 0433f776e5..be04834197 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -854,7 +854,7 @@ year_month {y}{e}{a}{r}_{m}{o}{n}{t}{h} return password {pow} lval.item = string(l.val) return pow -{power} lval.item = string(l.val) +{power} lval.item = string(l.val) return power {prepare} lval.item = string(l.val) return prepare From 218457791cbdde2dfef3047a873051bf68389491 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 11:07:51 +0800 Subject: [PATCH 24/63] builtin/time: Add tests for CURRENT_TIME(), CURTIME() --- expression/builtin/time_test.go | 9 +++++++++ mysql/time.go | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/expression/builtin/time_test.go b/expression/builtin/time_test.go index c3e662933f..f3da2d3ea8 100644 --- a/expression/builtin/time_test.go +++ b/expression/builtin/time_test.go @@ -275,3 +275,12 @@ func (s *testBuiltinSuite) TestCurrentDate(c *C) { c.Assert(ok, IsTrue) c.Assert(n.String(), GreaterEqual, last.Format(mysql.DateFormat)) } + +func (s *testBuiltinSuite) TestCurrentTime(c *C) { + last := time.Now() + v, err := builtinCurrentTime(nil, nil) + c.Assert(err, IsNil) + n, ok := v.(mysql.Time) + c.Assert(ok, IsTrue) + c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) +} diff --git a/mysql/time.go b/mysql/time.go index 446b2aaa19..85ab2c7085 100644 --- a/mysql/time.go +++ b/mysql/time.go @@ -34,9 +34,9 @@ var ( // Time format without fractional seconds precision. const ( - DateFormat = "2006-01-02" - CurtimeFormat = "15:04:05" - TimeFormat = "2006-01-02 15:04:05" + DateFormat = "2006-01-02" + CurrentTimeFormat = "15:04:05" + TimeFormat = "2006-01-02 15:04:05" // TimeFSPFormat is time format with fractional seconds precision. TimeFSPFormat = "2006-01-02 15:04:05.000000" ) @@ -127,7 +127,7 @@ func (t Time) String() string { } if t.Type == TypeDuration { - return t.Time.Format(CurtimeFormat) + return t.Time.Format(CurrentTimeFormat) } tfStr := TimeFormat From 8d0bf06cd536125242c597289c32e511d541591a Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 13:33:21 +0800 Subject: [PATCH 25/63] builtin/time: Add tests in parser and support the fsp argument --- expression/builtin/time.go | 17 +++++++++++++++-- mysql/time.go | 6 ++++-- parser/parser.y | 16 ++++++++++++---- parser/parser_test.go | 8 ++++++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/expression/builtin/time.go b/expression/builtin/time.go index 3c0f9b169d..d3cbe2582c 100644 --- a/expression/builtin/time.go +++ b/expression/builtin/time.go @@ -311,9 +311,22 @@ func builtinCurrentDate(args []interface{}, ctx map[interface{}]interface{}) (in // See https://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_curtime func builtinCurrentTime(args []interface{}, ctx map[interface{}]interface{}) (interface{}, error) { - return mysql.Time{ + fsp := 0 + if len(args) == 1 { + var err error + if fsp, err = checkFsp(args[0]); err != nil { + return nil, errors.Trace(err) + } + } + + t := mysql.Time{ Time: time.Now(), - Type: mysql.TypeDuration, Fsp: 0}, nil + Type: mysql.TypeDuration, + // set unspecified for later round + Fsp: mysql.UnspecifiedFsp, + } + + return t.RoundFrac(int(fsp)) } func checkFsp(arg interface{}) (int, error) { diff --git a/mysql/time.go b/mysql/time.go index 85ab2c7085..5a93c3fc08 100644 --- a/mysql/time.go +++ b/mysql/time.go @@ -126,11 +126,13 @@ func (t Time) String() string { return t.Time.Format(DateFormat) } + var tfStr string if t.Type == TypeDuration { - return t.Time.Format(CurrentTimeFormat) + tfStr = CurrentTimeFormat + } else { + tfStr = TimeFormat } - tfStr := TimeFormat if t.Fsp > 0 { tfStr = fmt.Sprintf("%s.%s", tfStr, strings.Repeat("0", t.Fsp)) } diff --git a/parser/parser.y b/parser/parser.y index 9c5d670d84..3456db7d3a 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -2139,13 +2139,21 @@ FunctionCallNonKeyword: { $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} } -| "CURTIME" '(' ')' +| "CURTIME" FuncDatetimePrec { - $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} + args := []ast.ExprNode{} + if $2 != nil { + args = append(args, $2.(ast.ExprNode)) + } + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: args} } -| "CURRENT_TIME" '(' ')' +| "CURRENT_TIME" FuncDatetimePrec { - $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} + args := []ast.ExprNode{} + if $2 != nil { + args = append(args, $2.(ast.ExprNode)) + } + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: args} } | "CURRENT_TIMESTAMP" FuncDatetimePrec { diff --git a/parser/parser_test.go b/parser/parser_test.go index d5bd73b7ba..81f87b810e 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -394,6 +394,14 @@ func (s *testParserSuite) TestBuiltin(c *C) { {"select now(6)", true}, {"select sysdate(), sysdate(6)", true}, + // Select current_time + {"select current_time", true}, + {"select current_time()", true}, + {"select current_time(6)", true}, + {"select curtime", true}, + {"select curtime()", true}, + {"select curtime(6)", true}, + // For time extract {`select extract(microsecond from "2011-11-11 10:10:10.123456")`, true}, {`select extract(second from "2011-11-11 10:10:10.123456")`, true}, From 8d938f32ab8e4734cad555cf41920efc28368e80 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 14:19:53 +0800 Subject: [PATCH 26/63] builtin/time: Add tests for the fsp argument in builtinCurrentTime() --- expression/builtin/time_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/expression/builtin/time_test.go b/expression/builtin/time_test.go index f3da2d3ea8..78fa0164af 100644 --- a/expression/builtin/time_test.go +++ b/expression/builtin/time_test.go @@ -282,5 +282,26 @@ func (s *testBuiltinSuite) TestCurrentTime(c *C) { c.Assert(err, IsNil) n, ok := v.(mysql.Time) c.Assert(ok, IsTrue) + c.Assert(n.String(), HasLen, 8) c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) + + v, err = builtinCurrentTime([]interface{}{3}, nil) + c.Assert(err, IsNil) + n, ok = v.(mysql.Time) + c.Assert(ok, IsTrue) + c.Assert(n.String(), HasLen, 12) + c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) + + v, err = builtinCurrentTime([]interface{}{6}, nil) + c.Assert(err, IsNil) + n, ok = v.(mysql.Time) + c.Assert(ok, IsTrue) + c.Assert(n.String(), HasLen, 15) + c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) + + v, err = builtinCurrentTime([]interface{}{-1}, nil) + c.Assert(err, NotNil) + + v, err = builtinCurrentTime([]interface{}{7}, nil) + c.Assert(err, NotNil) } From 78226701322fd1874018c94d0ab03725142b36e5 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 16:47:10 +0800 Subject: [PATCH 27/63] builtin/time: Use mysql.Duration instead of mysql.Time and add length check --- expression/builtin/time.go | 22 +++++++--------------- expression/builtin/time_test.go | 14 ++++++++------ mysql/time.go | 13 +++---------- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/expression/builtin/time.go b/expression/builtin/time.go index d3cbe2582c..69eec6e207 100644 --- a/expression/builtin/time.go +++ b/expression/builtin/time.go @@ -46,9 +46,9 @@ func convertToTime(arg interface{}, tp byte) (interface{}, error) { return t, nil } -func convertToDuration(arg interface{}) (interface{}, error) { +func convertToDuration(arg interface{}, fsp int) (interface{}, error) { f := types.NewFieldType(mysql.TypeDuration) - f.Decimal = mysql.MaxFsp + f.Decimal = fsp v, err := types.Convert(arg, f) if err != nil { @@ -79,7 +79,7 @@ func builtinDay(args []interface{}, ctx map[interface{}]interface{}) (interface{ // See http://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_hour func builtinHour(args []interface{}, ctx map[interface{}]interface{}) (interface{}, error) { - v, err := convertToDuration(args[0]) + v, err := convertToDuration(args[0], mysql.MaxFsp) if err != nil || types.IsNil(v) { return v, err } @@ -91,7 +91,7 @@ func builtinHour(args []interface{}, ctx map[interface{}]interface{}) (interface // See http://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_minute func builtinMinute(args []interface{}, ctx map[interface{}]interface{}) (interface{}, error) { - v, err := convertToDuration(args[0]) + v, err := convertToDuration(args[0], mysql.MaxFsp) if err != nil || types.IsNil(v) { return v, err } @@ -103,7 +103,7 @@ func builtinMinute(args []interface{}, ctx map[interface{}]interface{}) (interfa // See http://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_second func builtinSecond(args []interface{}, ctx map[interface{}]interface{}) (interface{}, error) { - v, err := convertToDuration(args[0]) + v, err := convertToDuration(args[0], mysql.MaxFsp) if err != nil || types.IsNil(v) { return v, err } @@ -115,7 +115,7 @@ func builtinSecond(args []interface{}, ctx map[interface{}]interface{}) (interfa // See http://dev.mysql.com/doc/refman/5.7/en/date-and-time-functions.html#function_microsecond func builtinMicroSecond(args []interface{}, ctx map[interface{}]interface{}) (interface{}, error) { - v, err := convertToDuration(args[0]) + v, err := convertToDuration(args[0], mysql.MaxFsp) if err != nil || types.IsNil(v) { return v, err } @@ -318,15 +318,7 @@ func builtinCurrentTime(args []interface{}, ctx map[interface{}]interface{}) (in return nil, errors.Trace(err) } } - - t := mysql.Time{ - Time: time.Now(), - Type: mysql.TypeDuration, - // set unspecified for later round - Fsp: mysql.UnspecifiedFsp, - } - - return t.RoundFrac(int(fsp)) + return convertToDuration(time.Now().Format("15:04:05.000000"), fsp) } func checkFsp(arg interface{}) (int, error) { diff --git a/expression/builtin/time_test.go b/expression/builtin/time_test.go index 78fa0164af..a7d75c3115 100644 --- a/expression/builtin/time_test.go +++ b/expression/builtin/time_test.go @@ -277,27 +277,29 @@ func (s *testBuiltinSuite) TestCurrentDate(c *C) { } func (s *testBuiltinSuite) TestCurrentTime(c *C) { + tfStr := "15:04:05" + last := time.Now() v, err := builtinCurrentTime(nil, nil) c.Assert(err, IsNil) - n, ok := v.(mysql.Time) + n, ok := v.(mysql.Duration) c.Assert(ok, IsTrue) c.Assert(n.String(), HasLen, 8) - c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) + c.Assert(n.String(), GreaterEqual, last.Format(tfStr)) v, err = builtinCurrentTime([]interface{}{3}, nil) c.Assert(err, IsNil) - n, ok = v.(mysql.Time) + n, ok = v.(mysql.Duration) c.Assert(ok, IsTrue) c.Assert(n.String(), HasLen, 12) - c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) + c.Assert(n.String(), GreaterEqual, last.Format(tfStr)) v, err = builtinCurrentTime([]interface{}{6}, nil) c.Assert(err, IsNil) - n, ok = v.(mysql.Time) + n, ok = v.(mysql.Duration) c.Assert(ok, IsTrue) c.Assert(n.String(), HasLen, 15) - c.Assert(n.String(), GreaterEqual, last.Format(mysql.CurrentTimeFormat)) + c.Assert(n.String(), GreaterEqual, last.Format(tfStr)) v, err = builtinCurrentTime([]interface{}{-1}, nil) c.Assert(err, NotNil) diff --git a/mysql/time.go b/mysql/time.go index 5a93c3fc08..6eb2c1f4cd 100644 --- a/mysql/time.go +++ b/mysql/time.go @@ -34,9 +34,8 @@ var ( // Time format without fractional seconds precision. const ( - DateFormat = "2006-01-02" - CurrentTimeFormat = "15:04:05" - TimeFormat = "2006-01-02 15:04:05" + DateFormat = "2006-01-02" + TimeFormat = "2006-01-02 15:04:05" // TimeFSPFormat is time format with fractional seconds precision. TimeFSPFormat = "2006-01-02 15:04:05.000000" ) @@ -126,13 +125,7 @@ func (t Time) String() string { return t.Time.Format(DateFormat) } - var tfStr string - if t.Type == TypeDuration { - tfStr = CurrentTimeFormat - } else { - tfStr = TimeFormat - } - + tfStr := TimeFormat if t.Fsp > 0 { tfStr = fmt.Sprintf("%s.%s", tfStr, strings.Repeat("0", t.Fsp)) } From 1009e715c91de7970f8b0445196bb0dbeb215d5d Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 17:28:21 +0800 Subject: [PATCH 28/63] builtin/time: Treat curTime as a synonym to currentTime and handle it at scanner part. --- parser/parser.y | 9 --------- parser/scanner.l | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/parser/parser.y b/parser/parser.y index 3456db7d3a..593be35b74 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -104,7 +104,6 @@ import ( cross "CROSS" curDate "CURDATE" currentDate "CURRENT_DATE" - curTime "CURTIME" currentTime "CURRENT_TIME" currentUser "CURRENT_USER" database "DATABASE" @@ -2139,14 +2138,6 @@ FunctionCallNonKeyword: { $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} } -| "CURTIME" FuncDatetimePrec - { - args := []ast.ExprNode{} - if $2 != nil { - args = append(args, $2.(ast.ExprNode)) - } - $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: args} - } | "CURRENT_TIME" FuncDatetimePrec { args := []ast.ExprNode{} diff --git a/parser/scanner.l b/parser/scanner.l index 9fdfd1fc6c..719b9005e3 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -695,7 +695,7 @@ year_month {y}{e}{a}{r}_{m}{o}{n}{t}{h} {current_date} lval.item = string(l.val) return currentDate {curtime} lval.item = string(l.val) - return curTime + return currentTime {current_time} lval.item = string(l.val) return currentTime {current_user} lval.item = string(l.val) From 0b70b41cfeb7ff6785ac37b0aa44bc42080ac9c2 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 19:29:12 +0800 Subject: [PATCH 29/63] builtin/time: fix one bug in function mapping The max arg should be 1, which means accept one argument. --- expression/builtin/builtin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expression/builtin/builtin.go b/expression/builtin/builtin.go index 0d266f3d4b..a214243938 100644 --- a/expression/builtin/builtin.go +++ b/expression/builtin/builtin.go @@ -69,9 +69,9 @@ var Funcs = map[string]Func{ // time functions "curdate": {builtinCurrentDate, 0, 0, false, false}, "current_date": {builtinCurrentDate, 0, 0, false, false}, - "current_time": {builtinCurrentTime, 0, 0, false, false}, + "current_time": {builtinCurrentTime, 0, 1, false, false}, "current_timestamp": {builtinNow, 0, 1, false, false}, - "curtime": {builtinCurrentTime, 0, 0, false, false}, + "curtime": {builtinCurrentTime, 0, 1, false, false}, "date": {builtinDate, 8, 8, true, false}, "day": {builtinDay, 1, 1, true, false}, "dayofmonth": {builtinDayOfMonth, 1, 1, true, false}, From a5b5fd67faced3ff5c0b1005264341d2a284246b Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Sun, 20 Dec 2015 23:17:17 +0800 Subject: [PATCH 30/63] evaluator: Check function call min and max arguments in new evaluator. For github issue #758. --- expression/builtin/builtin.go | 2 +- optimizer/evaluator/evaluator.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/expression/builtin/builtin.go b/expression/builtin/builtin.go index d5c73d5b4e..e6d7583a6f 100644 --- a/expression/builtin/builtin.go +++ b/expression/builtin/builtin.go @@ -70,7 +70,7 @@ var Funcs = map[string]Func{ "curdate": {builtinCurrentDate, 0, 0, false, false}, "current_date": {builtinCurrentDate, 0, 0, false, false}, "current_timestamp": {builtinNow, 0, 1, false, false}, - "date": {builtinDate, 8, 8, true, false}, + "date": {builtinDate, 1, 1, true, false}, "day": {builtinDay, 1, 1, true, false}, "dayofmonth": {builtinDayOfMonth, 1, 1, true, false}, "dayofweek": {builtinDayOfWeek, 1, 1, true, false}, diff --git a/optimizer/evaluator/evaluator.go b/optimizer/evaluator/evaluator.go index a3a0512c84..a9519e66ba 100644 --- a/optimizer/evaluator/evaluator.go +++ b/optimizer/evaluator/evaluator.go @@ -536,6 +536,10 @@ func (e *Evaluator) funcCall(v *ast.FuncCallExpr) bool { e.err = ErrInvalidOperation.Gen("unknown function %s", v.FnName.O) return false } + if len(v.Args) < f.MinArgs || len(v.Args) > f.MaxArgs { + e.err = ErrInvalidOperation.Gen("number of function arguments must in [%d, %d].", f.MinArgs, f.MaxArgs) + return false + } a := make([]interface{}, len(v.Args)) for i, arg := range v.Args { a[i] = arg.GetValue() From 9084d5d4657dce182476006b502c1cbab1c349bd Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 11:13:45 +0800 Subject: [PATCH 31/63] parser: fix parsing curtime 1. It can be used as an identifier 2. It should be called with parentheses --- parser/parser.y | 11 ++++++++++- parser/parser_test.go | 2 +- parser/scanner.l | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/parser/parser.y b/parser/parser.y index 593be35b74..b5eabf6555 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -104,6 +104,7 @@ import ( cross "CROSS" curDate "CURDATE" currentDate "CURRENT_DATE" + curTime "CUR_TIME" currentTime "CURRENT_TIME" currentUser "CURRENT_USER" database "DATABASE" @@ -1688,7 +1689,7 @@ NotKeywordToken: | "DAYOFWEEK" | "DAYOFYEAR" | "FOUND_ROWS" | "GROUP_CONCAT"| "HOUR" | "IFNULL" | "LENGTH" | "LOCATE" | "MAX" | "MICROSECOND" | "MIN" | "MINUTE" | "NULLIF" | "MONTH" | "NOW" | "RAND" | "SECOND" | "SQL_CALC_FOUND_ROWS" | "SUBDATE" | "SUBSTRING" %prec lowerThanLeftParen | "SUBSTRING_INDEX" | "SUM" | "TRIM" | "WEEKDAY" | "WEEKOFYEAR" -| "YEARWEEK" | "CONNECTION_ID" +| "YEARWEEK" | "CONNECTION_ID" | "CUR_TIME" /************************************************************************************ * @@ -2138,6 +2139,14 @@ FunctionCallNonKeyword: { $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string))} } +| "CUR_TIME" '(' ExpressionOpt ')' + { + args := []ast.ExprNode{} + if $3 != nil { + args = append(args, $3.(ast.ExprNode)) + } + $$ = &ast.FuncCallExpr{FnName: model.NewCIStr($1.(string)), Args: args} + } | "CURRENT_TIME" FuncDatetimePrec { args := []ast.ExprNode{} diff --git a/parser/parser_test.go b/parser/parser_test.go index 81f87b810e..7ccadaa78c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -41,6 +41,7 @@ func (s *testParserSuite) TestSimple(c *C) { "collation", "comment", "avg_row_length", "checksum", "compression", "connection", "key_block_size", "max_rows", "min_rows", "national", "row", "quarter", "escape", "grants", "status", "fields", "triggers", "delay_key_write", "isolation", "repeatable", "committed", "uncommitted", "only", "serializable", "level", + "curtime", } for _, kw := range unreservedKws { src := fmt.Sprintf("SELECT %s FROM tbl;", kw) @@ -398,7 +399,6 @@ func (s *testParserSuite) TestBuiltin(c *C) { {"select current_time", true}, {"select current_time()", true}, {"select current_time(6)", true}, - {"select curtime", true}, {"select curtime()", true}, {"select curtime(6)", true}, diff --git a/parser/scanner.l b/parser/scanner.l index 719b9005e3..9fdfd1fc6c 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -695,7 +695,7 @@ year_month {y}{e}{a}{r}_{m}{o}{n}{t}{h} {current_date} lval.item = string(l.val) return currentDate {curtime} lval.item = string(l.val) - return currentTime + return curTime {current_time} lval.item = string(l.val) return currentTime {current_user} lval.item = string(l.val) From 56a19f4e67676d1c41140c388bfa826598abc9fa Mon Sep 17 00:00:00 2001 From: disksing Date: Mon, 21 Dec 2015 11:25:37 +0800 Subject: [PATCH 32/63] localstore: add tests for engine.MultiSeek --- store/localstore/boltdb/boltdb_test.go | 10 ++++++++++ store/localstore/goleveldb/goleveldb_test.go | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/store/localstore/boltdb/boltdb_test.go b/store/localstore/boltdb/boltdb_test.go index 862c4eedc7..6bd749b251 100644 --- a/store/localstore/boltdb/boltdb_test.go +++ b/store/localstore/boltdb/boltdb_test.go @@ -138,4 +138,14 @@ func (s *testSuite) TestDB(c *C) { c.Assert(err, NotNil) c.Assert(k, IsNil) c.Assert(v, IsNil) + + m := db.MultiSeek([][]byte{[]byte("z"), []byte("a"), []byte("a1")}) + c.Assert(m, HasLen, 3) + c.Assert(m[0].Err, NotNil) + c.Assert(m[1].Err, IsNil) + c.Assert(m[1].Key, BytesEquals, []byte("a")) + c.Assert(m[1].Value, BytesEquals, []byte("1")) + c.Assert(m[2].Err, IsNil) + c.Assert(m[2].Key, BytesEquals, []byte("b")) + c.Assert(m[2].Value, BytesEquals, []byte("2")) } diff --git a/store/localstore/goleveldb/goleveldb_test.go b/store/localstore/goleveldb/goleveldb_test.go index 1e53166037..6d09cdf85f 100644 --- a/store/localstore/goleveldb/goleveldb_test.go +++ b/store/localstore/goleveldb/goleveldb_test.go @@ -95,4 +95,14 @@ func (s *testSuite) TestDB(c *C) { c.Assert(err, NotNil) c.Assert(k, IsNil) c.Assert(v, IsNil) + + m := db.MultiSeek([][]byte{[]byte("z"), []byte("a"), []byte("a1")}) + c.Assert(m, HasLen, 3) + c.Assert(m[0].Err, NotNil) + c.Assert(m[1].Err, IsNil) + c.Assert(m[1].Key, BytesEquals, []byte("a")) + c.Assert(m[1].Value, BytesEquals, []byte("2")) + c.Assert(m[2].Err, IsNil) + c.Assert(m[2].Key, BytesEquals, []byte("b")) + c.Assert(m[2].Value, BytesEquals, []byte("2")) } From a5dbd000dcd0ce1a8fe8fc39bd4298fe663f1e49 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Mon, 21 Dec 2015 11:45:44 +0800 Subject: [PATCH 33/63] evaluator: Add tests for funcCall. --- optimizer/evaluator/evaluator_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/optimizer/evaluator/evaluator_test.go b/optimizer/evaluator/evaluator_test.go index f634ce81db..a5892c7880 100644 --- a/optimizer/evaluator/evaluator_test.go +++ b/optimizer/evaluator/evaluator_test.go @@ -20,6 +20,8 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/ast" + "github.com/pingcap/tidb/expression/builtin" + "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/parser" "github.com/pingcap/tidb/parser/opcode" @@ -382,6 +384,30 @@ func (s *testEvaluatorSuite) TestConvert(c *C) { } } +func (s *testEvaluatorSuite) TestCall(c *C) { + // To test this case, we need to fake a wrong builtin.Funcs + builtin.Funcs = map[string]builtin.Func{ + "date": {nil, 8, 8, false, false}, + } + ctx := mock.NewContext() + + // Test case for unknown function + expr := &ast.FuncCallExpr{ + FnName: model.NewCIStr("unknown"), + Args: []ast.ExprNode{}, + } + _, err := Eval(ctx, expr) + c.Assert(err, NotNil) + + // Test case for invalid number of arguments + expr = &ast.FuncCallExpr{ + FnName: model.NewCIStr("date"), + Args: []ast.ExprNode{}, + } + _, err = Eval(ctx, expr) + c.Assert(err, NotNil) +} + func (s *testEvaluatorSuite) TestCast(c *C) { f := types.NewFieldType(mysql.TypeLonglong) From bb9630ce6c98b82a84999625a36e411b6b8ede25 Mon Sep 17 00:00:00 2001 From: disksing Date: Mon, 21 Dec 2015 11:56:08 +0800 Subject: [PATCH 34/63] *: address comment --- store/localstore/boltdb/boltdb_test.go | 30 ++++++++++++---- store/localstore/goleveldb/goleveldb_test.go | 36 ++++++++++++++------ 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/store/localstore/boltdb/boltdb_test.go b/store/localstore/boltdb/boltdb_test.go index 6bd749b251..54bbf0a27f 100644 --- a/store/localstore/boltdb/boltdb_test.go +++ b/store/localstore/boltdb/boltdb_test.go @@ -95,7 +95,7 @@ func (s *testSuite) TestPutNilAndDelete(c *C) { c.Assert(found, Equals, false) } -func (s *testSuite) TestDB(c *C) { +func (s *testSuite) TestGetSet(c *C) { db := s.db b := db.NewBatch() @@ -113,33 +113,49 @@ func (s *testSuite) TestDB(c *C) { v, err = db.Get([]byte("a")) c.Assert(err, IsNil) c.Assert(v, DeepEquals, []byte("1")) +} - k, v, err := db.Seek(nil) +func (s *testSuite) TestSeek(c *C) { + b := s.db.NewBatch() + b.Put([]byte("a"), []byte("1")) + b.Put([]byte("b"), []byte("2")) + err := s.db.Commit(b) + c.Assert(err, IsNil) + + k, v, err := s.db.Seek(nil) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("a")) c.Assert(v, BytesEquals, []byte("1")) - k, v, err = db.Seek([]byte("a")) + k, v, err = s.db.Seek([]byte("a")) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("a")) c.Assert(v, BytesEquals, []byte("1")) - k, v, err = db.Seek([]byte("b")) + k, v, err = s.db.Seek([]byte("b")) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("b")) c.Assert(v, BytesEquals, []byte("2")) - k, v, err = db.Seek([]byte("a1")) + k, v, err = s.db.Seek([]byte("a1")) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("b")) c.Assert(v, BytesEquals, []byte("2")) - k, v, err = db.Seek([]byte("c1")) + k, v, err = s.db.Seek([]byte("c1")) c.Assert(err, NotNil) c.Assert(k, IsNil) c.Assert(v, IsNil) +} - m := db.MultiSeek([][]byte{[]byte("z"), []byte("a"), []byte("a1")}) +func (s *testSuite) TestMultiSeek(c *C) { + b := s.db.NewBatch() + b.Put([]byte("a"), []byte("1")) + b.Put([]byte("b"), []byte("2")) + err := s.db.Commit(b) + c.Assert(err, IsNil) + + m := s.db.MultiSeek([][]byte{[]byte("z"), []byte("a"), []byte("a1")}) c.Assert(m, HasLen, 3) c.Assert(m[0].Err, NotNil) c.Assert(m[1].Err, IsNil) diff --git a/store/localstore/goleveldb/goleveldb_test.go b/store/localstore/goleveldb/goleveldb_test.go index 6d09cdf85f..23a3392902 100644 --- a/store/localstore/goleveldb/goleveldb_test.go +++ b/store/localstore/goleveldb/goleveldb_test.go @@ -43,7 +43,7 @@ func (s *testSuite) TearDownSuite(c *C) { s.db.Close() } -func (s *testSuite) TestDB(c *C) { +func (s *testSuite) TestGetSet(c *C) { db := s.db b := db.NewBatch() @@ -70,38 +70,54 @@ func (s *testSuite) TestDB(c *C) { v, err = db.Get([]byte("a")) c.Assert(err, IsNil) c.Assert(v, DeepEquals, []byte("2")) +} - k, v, err := db.Seek(nil) +func (s *testSuite) TestSeek(c *C) { + b := s.db.NewBatch() + b.Put([]byte("a"), []byte("1")) + b.Put([]byte("b"), []byte("2")) + err := s.db.Commit(b) + c.Assert(err, IsNil) + + k, v, err := s.db.Seek(nil) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("a")) - c.Assert(v, BytesEquals, []byte("2")) + c.Assert(v, BytesEquals, []byte("1")) - k, v, err = db.Seek([]byte("a")) + k, v, err = s.db.Seek([]byte("a")) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("a")) - c.Assert(v, BytesEquals, []byte("2")) + c.Assert(v, BytesEquals, []byte("1")) - k, v, err = db.Seek([]byte("b")) + k, v, err = s.db.Seek([]byte("b")) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("b")) c.Assert(v, BytesEquals, []byte("2")) - k, v, err = db.Seek([]byte("a1")) + k, v, err = s.db.Seek([]byte("a1")) c.Assert(err, IsNil) c.Assert(k, BytesEquals, []byte("b")) c.Assert(v, BytesEquals, []byte("2")) - k, v, err = db.Seek([]byte("c1")) + k, v, err = s.db.Seek([]byte("c1")) c.Assert(err, NotNil) c.Assert(k, IsNil) c.Assert(v, IsNil) +} - m := db.MultiSeek([][]byte{[]byte("z"), []byte("a"), []byte("a1")}) +func (s *testSuite) TestMultiSeek(c *C) { + b := s.db.NewBatch() + b.Put([]byte("a"), []byte("1")) + b.Put([]byte("b"), []byte("2")) + err := s.db.Commit(b) + c.Assert(err, IsNil) + + m := s.db.MultiSeek([][]byte{[]byte("z"), []byte("a"), []byte("a1")}) c.Assert(m, HasLen, 3) c.Assert(m[0].Err, NotNil) c.Assert(m[1].Err, IsNil) c.Assert(m[1].Key, BytesEquals, []byte("a")) - c.Assert(m[1].Value, BytesEquals, []byte("2")) + c.Assert(m[1].Value, BytesEquals, []byte("1")) c.Assert(m[2].Err, IsNil) c.Assert(m[2].Key, BytesEquals, []byte("b")) c.Assert(m[2].Value, BytesEquals, []byte("2")) From 2fe9938ca00e9e54fba78d6be4d662e8c5b237cf Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 12:15:56 +0800 Subject: [PATCH 35/63] *: Support show procedure status syntax --- ast/misc.go | 1 + executor/converter/convert_stmt.go | 2 ++ parser/parser.y | 7 +++++++ parser/parser_test.go | 3 ++- parser/scanner.l | 2 ++ plan/plans/show.go | 10 ++++++++++ stmt/stmt.go | 1 + 7 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ast/misc.go b/ast/misc.go index c5415122da..be45df476c 100644 --- a/ast/misc.go +++ b/ast/misc.go @@ -168,6 +168,7 @@ const ( ShowCreateTable ShowGrants ShowTriggers + ShowProcedureStatus ) // ShowStmt is a statement to provide information about databases, tables, columns and so on. diff --git a/executor/converter/convert_stmt.go b/executor/converter/convert_stmt.go index 6812416157..00fbb35345 100644 --- a/executor/converter/convert_stmt.go +++ b/executor/converter/convert_stmt.go @@ -878,6 +878,8 @@ func convertShow(converter *expressionConverter, v *ast.ShowStmt) (*stmts.ShowSt oldShow.Target = stmt.ShowTableStatus case ast.ShowTriggers: oldShow.Target = stmt.ShowTriggers + case ast.ShowProcedureStatus: + oldShow.Target = stmt.ShowProcedureStatus case ast.ShowNone: oldShow.Target = stmt.ShowNone } diff --git a/parser/parser.y b/parser/parser.y index b5eabf6555..d908fee1c1 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -214,6 +214,7 @@ import ( placeholder "PLACEHOLDER" prepare "PREPARE" primary "PRIMARY" + procedure "PROCEDURE" quarter "QUARTER" quick "QUICK" rand "RAND" @@ -3456,6 +3457,12 @@ ShowTargetFilterable: DBName: $2.(string), } } +| "PROCEDURE" "STATUS" + { + $$ = &ast.ShowStmt { + Tp: ast.ShowProcedureStatus, + } + } ShowLikeOrWhereOpt: { diff --git a/parser/parser_test.go b/parser/parser_test.go index 7ccadaa78c..b06336c29c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -262,7 +262,8 @@ func (s *testParserSuite) TestDMLStmt(c *C) { {`SHOW COLUMNS FROM City;`, true}, {`SHOW FIELDS FROM City;`, true}, {`SHOW TRIGGERS LIKE 't'`, true}, - {"SHOW DATABASES LIKE 'test2'", true}, + {`SHOW DATABASES LIKE 'test2'`, true}, + {`SHOW PROCEDURE STATUS WHERE Db='test'`, true}, // For default value {"CREATE TABLE sbtest (id INTEGER UNSIGNED NOT NULL AUTO_INCREMENT, k integer UNSIGNED DEFAULT '0' NOT NULL, c char(120) DEFAULT '' NOT NULL, pad char(60) DEFAULT '' NOT NULL, PRIMARY KEY (id) )", true}, diff --git a/parser/scanner.l b/parser/scanner.l index 9fdfd1fc6c..9e125165de 100644 --- a/parser/scanner.l +++ b/parser/scanner.l @@ -418,6 +418,7 @@ outer {o}{u}{t}{e}{r} password {p}{a}{s}{s}{w}{o}{r}{d} prepare {p}{r}{e}{p}{a}{r}{e} primary {p}{r}{i}{m}{a}{r}{y} +procedure {p}{r}{o}{c}{e}{d}{u}{r}{e} quarter {q}{u}{a}{r}{t}{e}{r} quick {q}{u}{i}{c}{k} rand {r}{a}{n}{d} @@ -859,6 +860,7 @@ year_month {y}{e}{a}{r}_{m}{o}{n}{t}{h} {prepare} lval.item = string(l.val) return prepare {primary} return primary +{procedure} return procedure {quarter} lval.item = string(l.val) return quarter {quick} lval.item = string(l.val) diff --git a/plan/plans/show.go b/plan/plans/show.go index 72ebaa5199..79910fd504 100644 --- a/plan/plans/show.go +++ b/plan/plans/show.go @@ -128,6 +128,9 @@ func (s *ShowPlan) GetFields() []*field.ResultField { "sql_mode", "Definer", "character_set_client", "collation_connection", "Database Collation"} types = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar} + case stmt.ShowProcedureStatus: + names = []string{} + types = []byte{} } fields := make([]*field.ResultField, 0, len(names)) for i, name := range names { @@ -193,6 +196,8 @@ func (s *ShowPlan) fetchAll(ctx context.Context) error { return s.fetchShowGrants(ctx) case stmt.ShowTriggers: return s.fetchShowTriggers(ctx) + case stmt.ShowProcedureStatus: + return s.fetchShowProcedureStatus(ctx) } return nil } @@ -660,3 +665,8 @@ func (s *ShowPlan) fetchShowGrants(ctx context.Context) error { func (s *ShowPlan) fetchShowTriggers(ctx context.Context) error { return nil } + +// Do nothing. +func (s *ShowPlan) fetchShowProcedureStatus(ctx context.Context) error { + return nil +} diff --git a/stmt/stmt.go b/stmt/stmt.go index e0ab1d5e05..d051b405cc 100644 --- a/stmt/stmt.go +++ b/stmt/stmt.go @@ -61,6 +61,7 @@ const ( ShowCreateTable ShowGrants ShowTriggers + ShowProcedureStatus ) const ( From 0539de52523cc70b20d21e3f64f9c5aec347bfd4 Mon Sep 17 00:00:00 2001 From: sllt Date: Mon, 21 Dec 2015 13:02:58 +0800 Subject: [PATCH 36/63] built-in/pow add test in parser and delete bultinPower function --- expression/builtin/builtin.go | 2 +- expression/builtin/math.go | 4 --- expression/builtin/math_test.go | 54 ++++++--------------------------- parser/parser.y | 2 +- parser/parser_test.go | 4 +++ 5 files changed, 15 insertions(+), 51 deletions(-) diff --git a/expression/builtin/builtin.go b/expression/builtin/builtin.go index 6f80b02d69..e17465e450 100644 --- a/expression/builtin/builtin.go +++ b/expression/builtin/builtin.go @@ -57,7 +57,7 @@ var Funcs = map[string]Func{ // math functions "abs": {builtinAbs, 1, 1, true, false}, "pow": {builtinPow, 2, 2, true, false}, - "power": {builtinPower, 2, 2, true, false}, + "power": {builtinPow, 2, 2, true, false}, "rand": {builtinRand, 0, 1, true, false}, // group by functions diff --git a/expression/builtin/math.go b/expression/builtin/math.go index dc2b04971b..cbf7c6be92 100644 --- a/expression/builtin/math.go +++ b/expression/builtin/math.go @@ -78,7 +78,3 @@ func builtinPow(args []interface{}, ctx map[interface{}]interface{}) (v interfac return math.Pow(x, y), nil } - -func builtinPower(args []interface{}, ctx map[interface{}]interface{}) (v interface{}, err error) { - return builtinPow(args, ctx) -} diff --git a/expression/builtin/math_test.go b/expression/builtin/math_test.go index 96621da301..d493fd01bc 100644 --- a/expression/builtin/math_test.go +++ b/expression/builtin/math_test.go @@ -61,51 +61,15 @@ func (s *testBuiltinSuite) TestPow(c *C) { c.Assert(v, DeepEquals, t.Ret) } - t := []interface{}{'a', 3} - _, err := builtinPow(t, nil) - c.Assert(err, NotNil) - t = []interface{}{3, 'a'} - _, err = builtinPow(t, nil) - c.Assert(err, NotNil) - - t = []interface{}{nil, 3} - _, err = builtinPow(t, nil) - c.Assert(err, NotNil) - t = []interface{}{3, nil} - _, err = builtinPow(t, nil) - c.Assert(err, NotNil) - -} - -func (s *testBuiltinSuite) TestPower(c *C) { - tbl := []struct { - Arg []interface{} - Ret float64 - }{ - {[]interface{}{1, 3}, 1}, - {[]interface{}{2, 2}, 4}, - {[]interface{}{4, 0.5}, 2}, - {[]interface{}{4, -2}, 0.0625}, + errTbl := [][]interface{}{ + []interface{}{"test", "test"}, + []interface{}{nil, nil}, + []interface{}{1, "test"}, + []interface{}{"test", 1}, + } + for _, t := range errTbl { + _, err := builtinPow(t, nil) + c.Assert(err, NotNil) } - for _, t := range tbl { - v, err := builtinPower(t.Arg, nil) - c.Assert(err, IsNil) - c.Assert(v, DeepEquals, t.Ret) - } - - t := []interface{}{'a', 3} - _, err := builtinPower(t, nil) - c.Assert(err, NotNil) - t = []interface{}{3, 'a'} - _, err = builtinPower(t, nil) - c.Assert(err, NotNil) - - t = []interface{}{nil, 3} - _, err = builtinPower(t, nil) - c.Assert(err, NotNil) - t = []interface{}{3, nil} - _, err = builtinPower(t, nil) - c.Assert(err, NotNil) - } diff --git a/parser/parser.y b/parser/parser.y index c797b0bb28..74d864139f 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -1687,7 +1687,7 @@ UnReservedKeyword: NotKeywordToken: "ABS" | "ADDDATE" | "COALESCE" | "CONCAT" | "CONCAT_WS" | "COUNT" | "DAY" | "DATE_ADD" | "DATE_SUB" | "DAYOFMONTH" | "DAYOFWEEK" | "DAYOFYEAR" | "FOUND_ROWS" | "GROUP_CONCAT"| "HOUR" | "IFNULL" | "LENGTH" | "LOCATE" | "MAX" -| "MICROSECOND" | "MIN" | "MINUTE" | "NULLIF" | "MONTH" | "NOW" | "RAND" | "SECOND" | "SQL_CALC_FOUND_ROWS" +| "MICROSECOND" | "MIN" | "MINUTE" | "NULLIF" | "MONTH" | "NOW" | "POW" | "POWER" | "RAND" | "SECOND" | "SQL_CALC_FOUND_ROWS" | "SUBDATE" | "SUBSTRING" %prec lowerThanLeftParen | "SUBSTRING_INDEX" | "SUM" | "TRIM" | "WEEKDAY" | "WEEKOFYEAR" | "YEARWEEK" | "CONNECTION_ID" diff --git a/parser/parser_test.go b/parser/parser_test.go index d5bd73b7ba..8ab3906b00 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -330,6 +330,10 @@ func (s *testParserSuite) TestBuiltin(c *C) { table := []testCase{ // For buildin functions {"SELECT DAYOFMONTH('2007-02-03');", true}, + {"SELECT POW(1,2)", true}, + {"SELECT POW(1, 0.5)", true}, + {"SELECT POW(1, -1)", true}, + {"SELECT POW(-1, 1)", true}, {"SELECT RAND();", true}, {"SELECT RAND(1);", true}, From abc4701b00a56c2319258b2a0733091fa6301835 Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Mon, 21 Dec 2015 13:10:40 +0800 Subject: [PATCH 37/63] executor: make seek value with the same length of the index columns. For composite index with 2 columns, if we just pass one value in `Seek`, we will get the wrong result. So this change create the seek value with the length of the composite index. --- executor/builder.go | 2 +- executor/executor.go | 10 +++++----- kv/index_iter_test.go | 20 ++++++++++++++++++++ session_test.go | 7 +++++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/executor/builder.go b/executor/builder.go index 5848120299..41691995c1 100644 --- a/executor/builder.go +++ b/executor/builder.go @@ -84,7 +84,7 @@ func (b *executorBuilder) buildIndexScan(v *plan.IndexScan) Executor { } e := &IndexScanExec{ tbl: tbl, - idx: idx.X, + idx: idx, fields: v.Fields(), ctx: b.ctx, Desc: v.Desc, diff --git a/executor/executor.go b/executor/executor.go index a00940672e..50fc6cace9 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -19,6 +19,7 @@ import ( "github.com/juju/errors" "github.com/pingcap/tidb/ast" + "github.com/pingcap/tidb/column" "github.com/pingcap/tidb/context" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/optimizer/evaluator" @@ -177,8 +178,8 @@ func (e *IndexRangeExec) Fields() []*ast.ResultField { // Next implements Executor Next interface. func (e *IndexRangeExec) Next() (*Row, error) { if e.iter == nil { - seekVals := make([]interface{}, len(e.lowVals)) - for i := 0; i < len(seekVals); i++ { + seekVals := make([]interface{}, len(e.scan.idx.Columns)) + for i := 0; i < len(e.lowVals); i++ { var err error if e.lowVals[i] == plan.MinNotNullVal { seekVals[i] = []byte{} @@ -189,12 +190,11 @@ func (e *IndexRangeExec) Next() (*Row, error) { } } } - txn, err := e.scan.ctx.GetTxn(false) if err != nil { return nil, errors.Trace(err) } - e.iter, _, err = e.scan.idx.Seek(txn, seekVals) + e.iter, _, err = e.scan.idx.X.Seek(txn, seekVals) if err != nil { return nil, types.EOFAsNil(err) } @@ -315,7 +315,7 @@ func (e *IndexRangeExec) Close() error { // IndexScanExec represents an index scan executor. type IndexScanExec struct { tbl table.Table - idx kv.Index + idx *column.IndexedCol fields []*ast.ResultField Ranges []*IndexRangeExec Desc bool diff --git a/kv/index_iter_test.go b/kv/index_iter_test.go index 969e2093d3..4d92eb2ff8 100644 --- a/kv/index_iter_test.go +++ b/kv/index_iter_test.go @@ -143,3 +143,23 @@ func (s *testIndexSuite) TestIndex(c *C) { err = txn.Commit() c.Assert(err, IsNil) } + +func (s *testIndexSuite) TestCombineIndexSeek(c *C) { + index := kv.NewKVIndex("i", "test", 1, false) + + txn, err := s.s.Begin() + c.Assert(err, IsNil) + + values := []interface{}{"abc", "def"} + err = index.Create(txn, values, 1) + c.Assert(err, IsNil) + + index2 := kv.NewKVIndex("i", "test", 1, false) + iter, hit, err := index2.Seek(txn, []interface{}{"abc", nil}) + defer iter.Close() + c.Assert(err, IsNil) + c.Assert(hit, IsFalse) + _, h, err := iter.Next() + c.Assert(err, IsNil) + c.Assert(h, Equals, int64(1)) +} diff --git a/session_test.go b/session_test.go index 7bfbaf8376..8021ac46a7 100644 --- a/session_test.go +++ b/session_test.go @@ -1364,6 +1364,13 @@ func (s *testSessionSuite) TestMultiColumnIndex(c *C) { checkPlan(c, se, sql, expectedExplain) mustExecMatch(c, se, sql, [][]interface{}{{1}}) + // Test varchar type. + mustExecSQL(c, se, "drop table t;") + mustExecSQL(c, se, "create table t (c1 varchar(64), c2 varchar(64), index c1_c2 (c1, c2));") + mustExecSQL(c, se, "insert into t values ('abc', 'def')") + sql = "select c1 from t where c1 = 'abc'" + mustExecMatch(c, se, sql, [][]interface{}{{[]byte("abc")}}) + err := se.Close() c.Assert(err, IsNil) } From 5d9e7bb4fc7520a1cd5278caa1188aabf08bcf07 Mon Sep 17 00:00:00 2001 From: sllt Date: Mon, 21 Dec 2015 13:30:01 +0800 Subject: [PATCH 38/63] fix built-in/pow test --- expression/builtin/math_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/expression/builtin/math_test.go b/expression/builtin/math_test.go index d493fd01bc..364c7ae491 100644 --- a/expression/builtin/math_test.go +++ b/expression/builtin/math_test.go @@ -61,14 +61,16 @@ func (s *testBuiltinSuite) TestPow(c *C) { c.Assert(v, DeepEquals, t.Ret) } - errTbl := [][]interface{}{ - []interface{}{"test", "test"}, - []interface{}{nil, nil}, - []interface{}{1, "test"}, - []interface{}{"test", 1}, + errTbl := []struct { + Arg []interface{} + }{ + {[]interface{}{"test", "test"}}, + {[]interface{}{nil, nil}}, + {[]interface{}{1, "test"}}, + {[]interface{}{1, nil}}, } for _, t := range errTbl { - _, err := builtinPow(t, nil) + _, err := builtinPow(t.Arg, nil) c.Assert(err, NotNil) } From 48d2d87a9586249ffe5273d0c6a745e59a94354c Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 13:38:09 +0800 Subject: [PATCH 39/63] plans: Fix bug in show full tables --- plan/plans/info_test.go | 7 +++++++ plan/plans/show.go | 12 ++++++++++-- plan/plans/show_test.go | 7 ++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/plan/plans/info_test.go b/plan/plans/info_test.go index 5067b7529f..94284d8922 100644 --- a/plan/plans/info_test.go +++ b/plan/plans/info_test.go @@ -68,6 +68,13 @@ func mustQuery(c *C, currDB *sql.DB, s string) int { return cnt } +func mustFailQuery(c *C, currDB *sql.DB, s string) { + rows, _ := currDB.Query(s) + rows.Next() + c.Assert(rows.Err(), NotNil) + rows.Close() +} + func mustExec(c *C, currDB *sql.DB, sql string) sql.Result { tx := mustBegin(c, currDB) r := mustExecuteSql(c, tx, sql) diff --git a/plan/plans/show.go b/plan/plans/show.go index 72ebaa5199..c51fe21b5b 100644 --- a/plan/plans/show.go +++ b/plan/plans/show.go @@ -356,8 +356,16 @@ func (s *ShowPlan) fetchShowTables(ctx context.Context) error { s.Pattern.Expr = expression.Value{Val: data[0]} } else if s.Where != nil { m[expression.ExprEvalIdentFunc] = func(name string) (interface{}, error) { - if s.Full && strings.EqualFold(name, "Table_type") { - return data[1], nil + // The first column is Tables_in_{database}. + // If s.Full is true, there is a column named Table_type at the second place. + if strings.EqualFold(name, "Table_type") { + if s.Full { + return data[1], nil + } + } + fieldName := fmt.Sprintf("Tables_in_%s", dbName) + if strings.EqualFold(name, fieldName) { + return data[0], nil } return nil, errors.Errorf("unknown field %s", name) } diff --git a/plan/plans/show_test.go b/plan/plans/show_test.go index 8f63a1f87e..7c5e83bcb2 100644 --- a/plan/plans/show_test.go +++ b/plan/plans/show_test.go @@ -360,11 +360,12 @@ func (p *testShowSuit) TestShowTables(c *C) { c.Assert(cnt, Equals, 2) cnt = mustQuery(c, testDB, `show full tables where Table_type != 'VIEW';`) c.Assert(cnt, Equals, 3) + cnt = mustQuery(c, testDB, `show full tables where Tables_in_test='tab00' and Table_type != 'VIEW';`) + c.Assert(cnt, Equals, 1) + mustFailQuery(c, testDB, `show full tables where Tables_in_unknowndb ='tab00' and Table_type != 'VIEW';`) mustQuery(c, testDB, `show create table tab00;`) - rows, _ := testDB.Query(`show create table abc;`) - rows.Next() - c.Assert(rows.Err(), NotNil) + mustFailQuery(c, testDB, `show create table abc;`) } func (p *testShowSuit) TestShowGrants(c *C) { From 55d12c184fda7b925a142296c069633523da2111 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Mon, 21 Dec 2015 14:45:51 +0800 Subject: [PATCH 40/63] evaluator: Refine tests for funcCall. 1. Remove the fake builtin.Func. 2. Fix one bug in handling unlimited arguments upper bound. --- optimizer/evaluator/evaluator.go | 2 +- optimizer/evaluator/evaluator_test.go | 33 ++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/optimizer/evaluator/evaluator.go b/optimizer/evaluator/evaluator.go index a9519e66ba..ec46727100 100644 --- a/optimizer/evaluator/evaluator.go +++ b/optimizer/evaluator/evaluator.go @@ -536,7 +536,7 @@ func (e *Evaluator) funcCall(v *ast.FuncCallExpr) bool { e.err = ErrInvalidOperation.Gen("unknown function %s", v.FnName.O) return false } - if len(v.Args) < f.MinArgs || len(v.Args) > f.MaxArgs { + if len(v.Args) < f.MinArgs || (f.MaxArgs != -1 && len(v.Args) > f.MaxArgs) { e.err = ErrInvalidOperation.Gen("number of function arguments must in [%d, %d].", f.MinArgs, f.MaxArgs) return false } diff --git a/optimizer/evaluator/evaluator_test.go b/optimizer/evaluator/evaluator_test.go index a5892c7880..b30f2db273 100644 --- a/optimizer/evaluator/evaluator_test.go +++ b/optimizer/evaluator/evaluator_test.go @@ -20,7 +20,6 @@ import ( . "github.com/pingcap/check" "github.com/pingcap/tidb/ast" - "github.com/pingcap/tidb/expression/builtin" "github.com/pingcap/tidb/model" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/parser" @@ -385,10 +384,6 @@ func (s *testEvaluatorSuite) TestConvert(c *C) { } func (s *testEvaluatorSuite) TestCall(c *C) { - // To test this case, we need to fake a wrong builtin.Funcs - builtin.Funcs = map[string]builtin.Func{ - "date": {nil, 8, 8, false, false}, - } ctx := mock.NewContext() // Test case for unknown function @@ -399,13 +394,39 @@ func (s *testEvaluatorSuite) TestCall(c *C) { _, err := Eval(ctx, expr) c.Assert(err, NotNil) - // Test case for invalid number of arguments + // Test case for invalid number of arguments, violating the lower bound expr = &ast.FuncCallExpr{ FnName: model.NewCIStr("date"), Args: []ast.ExprNode{}, } _, err = Eval(ctx, expr) c.Assert(err, NotNil) + + // Test case for invalid number of arguments, violating the upper bound + expr = &ast.FuncCallExpr{ + FnName: model.NewCIStr("date"), + Args: []ast.ExprNode{ast.NewValueExpr("2015-12-21"), + ast.NewValueExpr("2015-12-22")}, + } + _, err = Eval(ctx, expr) + c.Assert(err, NotNil) + + // Test case for correct number of arguments + expr = &ast.FuncCallExpr{ + FnName: model.NewCIStr("date"), + Args: []ast.ExprNode{ast.NewValueExpr("2015-12-21")}, + } + _, err = Eval(ctx, expr) + c.Assert(err, IsNil) + + // Test case for unlimited upper bound + expr = &ast.FuncCallExpr{ + FnName: model.NewCIStr("concat"), + Args: []ast.ExprNode{ast.NewValueExpr(1), + ast.NewValueExpr(2)}, + } + _, err = Eval(ctx, expr) + c.Assert(err, IsNil) } func (s *testEvaluatorSuite) TestCast(c *C) { From 4f551e17da157aec55b237b6dee289e30b3899a1 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Mon, 21 Dec 2015 15:13:09 +0800 Subject: [PATCH 41/63] evaluator: Check return values from valid call expressions in funcCall test. --- optimizer/evaluator/evaluator_test.go | 42 +++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/optimizer/evaluator/evaluator_test.go b/optimizer/evaluator/evaluator_test.go index b30f2db273..c6f87366a7 100644 --- a/optimizer/evaluator/evaluator_test.go +++ b/optimizer/evaluator/evaluator_test.go @@ -386,12 +386,33 @@ func (s *testEvaluatorSuite) TestConvert(c *C) { func (s *testEvaluatorSuite) TestCall(c *C) { ctx := mock.NewContext() - // Test case for unknown function + // Test case for correct number of arguments expr := &ast.FuncCallExpr{ + FnName: model.NewCIStr("date"), + Args: []ast.ExprNode{ast.NewValueExpr("2015-12-21 11:11:11")}, + } + v, err := Eval(ctx, expr) + c.Assert(err, IsNil) + value, ok := v.(mysql.Time) + c.Assert(ok, IsTrue) + c.Assert(value.String(), Equals, "2015-12-21") + + // Test case for unlimited upper bound + expr = &ast.FuncCallExpr{ + FnName: model.NewCIStr("concat"), + Args: []ast.ExprNode{ast.NewValueExpr("Ti"), + ast.NewValueExpr("D"), ast.NewValueExpr("B")}, + } + v, err = Eval(ctx, expr) + c.Assert(err, IsNil) + c.Assert(v, Equals, "TiDB") + + // Test case for unknown function + expr = &ast.FuncCallExpr{ FnName: model.NewCIStr("unknown"), Args: []ast.ExprNode{}, } - _, err := Eval(ctx, expr) + _, err = Eval(ctx, expr) c.Assert(err, NotNil) // Test case for invalid number of arguments, violating the lower bound @@ -410,23 +431,6 @@ func (s *testEvaluatorSuite) TestCall(c *C) { } _, err = Eval(ctx, expr) c.Assert(err, NotNil) - - // Test case for correct number of arguments - expr = &ast.FuncCallExpr{ - FnName: model.NewCIStr("date"), - Args: []ast.ExprNode{ast.NewValueExpr("2015-12-21")}, - } - _, err = Eval(ctx, expr) - c.Assert(err, IsNil) - - // Test case for unlimited upper bound - expr = &ast.FuncCallExpr{ - FnName: model.NewCIStr("concat"), - Args: []ast.ExprNode{ast.NewValueExpr(1), - ast.NewValueExpr(2)}, - } - _, err = Eval(ctx, expr) - c.Assert(err, IsNil) } func (s *testEvaluatorSuite) TestCast(c *C) { From 186f0cb3ee3dd789023e71d1f8e68d55beff0bcb Mon Sep 17 00:00:00 2001 From: sllt Date: Mon, 21 Dec 2015 16:11:05 +0800 Subject: [PATCH 42/63] built-in/pow add test in parser --- parser/parser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 8e442c8dd5..57da97e210 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -331,7 +331,7 @@ func (s *testParserSuite) TestBuiltin(c *C) { table := []testCase{ // For buildin functions {"SELECT DAYOFMONTH('2007-02-03');", true}, - {"SELECT POW(1,2)", true}, + {"SELECT POW(1, 2)", true}, {"SELECT POW(1, 0.5)", true}, {"SELECT POW(1, -1)", true}, {"SELECT POW(-1, 1)", true}, From 48fb16316b92b681cae7d785a8c8e5f978031a37 Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 16:28:19 +0800 Subject: [PATCH 43/63] plans: Address comment --- plan/plans/show.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plan/plans/show.go b/plan/plans/show.go index c51fe21b5b..6214c974fc 100644 --- a/plan/plans/show.go +++ b/plan/plans/show.go @@ -357,11 +357,9 @@ func (s *ShowPlan) fetchShowTables(ctx context.Context) error { } else if s.Where != nil { m[expression.ExprEvalIdentFunc] = func(name string) (interface{}, error) { // The first column is Tables_in_{database}. - // If s.Full is true, there is a column named Table_type at the second place. - if strings.EqualFold(name, "Table_type") { - if s.Full { - return data[1], nil - } + // If s.Full is true, there will be a column named Table_type at the second place. + if s.Full && strings.EqualFold(name, "Table_type") { + return data[1], nil } fieldName := fmt.Sprintf("Tables_in_%s", dbName) if strings.EqualFold(name, fieldName) { From 290383df5e8eef2d934e97921899a8516b9bb665 Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Mon, 21 Dec 2015 16:53:30 +0800 Subject: [PATCH 44/63] store/hbase: fix build. --- store/hbase/kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/hbase/kv.go b/store/hbase/kv.go index b19bd805fe..4844432392 100644 --- a/store/hbase/kv.go +++ b/store/hbase/kv.go @@ -172,7 +172,7 @@ func (d Driver) Open(dsn string) (kv.Storage, error) { } if !b { // Create new hbase table for store. - t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName)) + t := hbase.NewTableDesciptor(tableName) cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily) cf.AddAttr("THEMIS_ENABLE", "true") t.AddColumnDesc(cf) From 7b4a3da9b78a9b80a6415aecbeb97bc6a29046b3 Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 17:56:26 +0800 Subject: [PATCH 45/63] plans: Address comment --- plan/plans/info_test.go | 3 ++- plan/plans/show_test.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/plan/plans/info_test.go b/plan/plans/info_test.go index 94284d8922..405e7acbbd 100644 --- a/plan/plans/info_test.go +++ b/plan/plans/info_test.go @@ -69,7 +69,8 @@ func mustQuery(c *C, currDB *sql.DB, s string) int { } func mustFailQuery(c *C, currDB *sql.DB, s string) { - rows, _ := currDB.Query(s) + rows, err := currDB.Query(s) + c.Assert(err, IsNil) rows.Next() c.Assert(rows.Err(), NotNil) rows.Close() diff --git a/plan/plans/show_test.go b/plan/plans/show_test.go index 7c5e83bcb2..041075a405 100644 --- a/plan/plans/show_test.go +++ b/plan/plans/show_test.go @@ -363,6 +363,7 @@ func (p *testShowSuit) TestShowTables(c *C) { cnt = mustQuery(c, testDB, `show full tables where Tables_in_test='tab00' and Table_type != 'VIEW';`) c.Assert(cnt, Equals, 1) mustFailQuery(c, testDB, `show full tables where Tables_in_unknowndb ='tab00' and Table_type != 'VIEW';`) + mustFailQuery(c, testDB, `show tables where Tables_in_test ='tab00' and Table_type != 'VIEW';`) mustQuery(c, testDB, `show create table tab00;`) mustFailQuery(c, testDB, `show create table abc;`) From 818fafd45a40b86e16848f732d86ec5db11356be Mon Sep 17 00:00:00 2001 From: ngaut Date: Mon, 21 Dec 2015 21:26:22 +0800 Subject: [PATCH 46/63] boltdb:Make MultiSeek faster --- store/localstore/boltdb/boltdb.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/store/localstore/boltdb/boltdb.go b/store/localstore/boltdb/boltdb.go index bbc4d95d24..ad1ec4de1c 100644 --- a/store/localstore/boltdb/boltdb.go +++ b/store/localstore/boltdb/boltdb.go @@ -53,11 +53,27 @@ func (d *db) Get(key []byte) ([]byte, error) { func (d *db) MultiSeek(keys [][]byte) []*engine.MSeekResult { res := make([]*engine.MSeekResult, 0, len(keys)) - for _, key := range keys { - r := &engine.MSeekResult{} - r.Key, r.Value, r.Err = d.Seek(key) - res = append(res, r) - } + d.DB.View(func(tx *bolt.Tx) error { + b := tx.Bucket(bucketName) + c := b.Cursor() + for _, key := range keys { + var k, v []byte + if key == nil { + k, v = c.First() + } else { + k, v = c.Seek(key) + } + + r := &engine.MSeekResult{} + r.Key, r.Value, r.Err = bytes.CloneBytes(k), bytes.CloneBytes(v), nil + if k == nil { + r.Err = engine.ErrNotFound + } + res = append(res, r) + } + return nil + }) + return res } From 2015287078954b1b5b6ba11bc71095c05d2217b1 Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 22:37:51 +0800 Subject: [PATCH 47/63] *: Fix bug in convert_expr.go that miss SubstringIndexExpr --- executor/converter/convert_expr.go | 11 +++++++++++ session_test.go | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/executor/converter/convert_expr.go b/executor/converter/convert_expr.go index 7d395b8361..1206952707 100644 --- a/executor/converter/convert_expr.go +++ b/executor/converter/convert_expr.go @@ -106,6 +106,8 @@ func (c *expressionConverter) Leave(in ast.Node) (out ast.Node, ok bool) { c.funcCast(v) case *ast.FuncSubstringExpr: c.funcSubstring(v) + case *ast.FuncSubstringIndexExpr: + c.funcSubstringIndex(v) case *ast.FuncLocateExpr: c.funcLocate(v) case *ast.FuncTrimExpr: @@ -363,6 +365,15 @@ func (c *expressionConverter) funcSubstring(v *ast.FuncSubstringExpr) { c.exprMap[v] = oldSubstring } +func (c *expressionConverter) funcSubstringIndex(v *ast.FuncSubstringIndexExpr) { + oldSubstrIdx := &expression.FunctionSubstringIndex{ + Delim: c.exprMap[v.Delim], + Count: c.exprMap[v.Count], + StrExpr: c.exprMap[v.StrExpr], + } + c.exprMap[v] = oldSubstrIdx +} + func (c *expressionConverter) funcLocate(v *ast.FuncLocateExpr) { oldLocate := &expression.FunctionLocate{ Pos: c.exprMap[v.Pos], diff --git a/session_test.go b/session_test.go index 7bfbaf8376..18ad9f5ffd 100644 --- a/session_test.go +++ b/session_test.go @@ -1368,6 +1368,15 @@ func (s *testSessionSuite) TestMultiColumnIndex(c *C) { c.Assert(err, IsNil) } +func (s *testSessionSuite) TestSubstringIndexExpr(c *C) { + store := newStore(c, s.dbName) + se := newSession(c, store, s.dbName) + mustExecSQL(c, se, "drop table if exists t;") + mustExecSQL(c, se, "create table t (c varchar(128));") + mustExecSQL(c, se, `insert into t values ("www.mysql.com");`) + mustExecMatch(c, se, "SELECT DISTINCT SUBSTRING_INDEX(c, '.', 2) from t;", [][]interface{}{{"www.mysql"}}) +} + func (s *testSessionSuite) TestGlobalVarAccessor(c *C) { varName := "max_allowed_packet" From cfe157f93fe2db5afb956d34fa32cff716f56386 Mon Sep 17 00:00:00 2001 From: Yanzhe Chen Date: Mon, 21 Dec 2015 23:03:39 +0800 Subject: [PATCH 48/63] *: Unify session ID and connection ID This is for Github issue #722. 1. CONNECTION_ID now can get the real id generate at server side (rather than a session id). 2. Keep session id generation untouched to ensure minimal interface change. --- expression/builtin/info.go | 22 +--------------------- expression/builtin/info_test.go | 6 ++++-- session.go | 9 +++++---- sessionctx/variable/session.go | 3 +++ tidb-server/server/conn.go | 2 +- tidb-server/server/driver.go | 4 ++-- tidb-server/server/driver_tidb.go | 3 ++- 7 files changed, 18 insertions(+), 31 deletions(-) diff --git a/expression/builtin/info.go b/expression/builtin/info.go index 3e5e31a091..14687d6b42 100644 --- a/expression/builtin/info.go +++ b/expression/builtin/info.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/tidb/context" "github.com/pingcap/tidb/sessionctx/db" "github.com/pingcap/tidb/sessionctx/variable" - "github.com/pingcap/tidb/terror" ) // See: https://dev.mysql.com/doc/refman/5.7/en/information-functions.html @@ -69,30 +68,11 @@ func builtinUser(args []interface{}, data map[interface{}]interface{}) (v interf return variable.GetSessionVars(ctx).User, nil } -// connectionIDKeyType is a dummy type to avoid naming collision in context. -type connectionIDKeyType int - -// String defines a Stringer function for debugging and pretty printing. -func (k connectionIDKeyType) String() string { - return "connection_id" -} - -// ConnectionIDKey is the key for get connection id from context -const ConnectionIDKey connectionIDKeyType = 0 - func builtinConnectionID(args []interface{}, data map[interface{}]interface{}) (v interface{}, err error) { c, ok := data[ExprEvalArgCtx] if !ok { return nil, errors.Errorf("Missing ExprEvalArgCtx when evalue builtin") } ctx := c.(context.Context) - idValue := ctx.Value(ConnectionIDKey) - if idValue == nil { - return nil, terror.MissConnectionID - } - id, ok := idValue.(int64) - if !ok { - return nil, terror.MissConnectionID.Gen("connection id is not int64 but %T", idValue) - } - return id, nil + return variable.GetSessionVars(ctx).ConnectionID, nil } diff --git a/expression/builtin/info_test.go b/expression/builtin/info_test.go index 260596aff8..93eca4577b 100644 --- a/expression/builtin/info_test.go +++ b/expression/builtin/info_test.go @@ -86,9 +86,11 @@ func (s *testBuiltinSuite) TestConnectionID(c *C) { ctx := mock.NewContext() m := map[interface{}]interface{}{} variable.BindSessionVars(ctx) - ctx.SetValue(ConnectionIDKey, int64(1)) + sessionVars := variable.GetSessionVars(ctx) + sessionVars.ConnectionID = uint64(1) + m[ExprEvalArgCtx] = ctx v, err := builtinConnectionID(nil, m) c.Assert(err, IsNil) - c.Assert(v, Equals, int64(1)) + c.Assert(v, Equals, uint64(1)) } diff --git a/session.go b/session.go index 730b18e617..5fd018cef8 100644 --- a/session.go +++ b/session.go @@ -29,7 +29,6 @@ import ( "github.com/juju/errors" "github.com/ngaut/log" "github.com/pingcap/tidb/context" - "github.com/pingcap/tidb/expression/builtin" "github.com/pingcap/tidb/field" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" @@ -65,6 +64,7 @@ type Session interface { ExecutePreparedStmt(stmtID uint32, param ...interface{}) (rset.Recordset, error) DropPreparedStmt(stmtID uint32) error SetClientCapability(uint32) // Set client capability flags + SetConnectionID(uint64) Close() error Retry() error Auth(user string, auth []byte, salt []byte) bool @@ -145,6 +145,10 @@ func (s *session) SetClientCapability(capability uint32) { variable.GetSessionVars(s).ClientCapability = capability } +func (s *session) SetConnectionID(connectionID uint64) { + variable.GetSessionVars(s).ConnectionID = connectionID +} + func (s *session) FinishTxn(rollback bool) error { // transaction has already been committed or rolled back if s.txn == nil { @@ -580,9 +584,6 @@ func CreateSession(store kv.Storage) (Session, error) { variable.BindSessionVars(s) variable.GetSessionVars(s).SetStatusFlag(mysql.ServerStatusAutocommit, true) - // set connection id - s.SetValue(builtin.ConnectionIDKey, s.sid) - // session implements variable.GlobalVarAccessor. Bind it to ctx. variable.BindGlobalVarAccessor(s, s) diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 23c04e0ebb..d0c3acfb9a 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -36,6 +36,9 @@ type SessionVars struct { // Client capability ClientCapability uint32 + // Connection ID + ConnectionID uint64 + // Found rows FoundRows uint64 diff --git a/tidb-server/server/conn.go b/tidb-server/server/conn.go index 4b7e0503b0..3943173b17 100644 --- a/tidb-server/server/conn.go +++ b/tidb-server/server/conn.go @@ -191,7 +191,7 @@ func (cc *clientConn) readHandshakeResponse() error { } } // Open session and do auth - cc.ctx, err = cc.server.driver.OpenCtx(cc.capability, uint8(cc.collation), cc.dbname) + cc.ctx, err = cc.server.driver.OpenCtx(uint64(cc.connectionID), cc.capability, uint8(cc.collation), cc.dbname) if err != nil { cc.Close() return errors.Trace(err) diff --git a/tidb-server/server/driver.go b/tidb-server/server/driver.go index 1f1c9a15ff..d00444c46c 100644 --- a/tidb-server/server/driver.go +++ b/tidb-server/server/driver.go @@ -15,8 +15,8 @@ package server // IDriver opens IContext. type IDriver interface { - // OpenCtx opens an IContext with client capability, collation and dbname. - OpenCtx(capability uint32, collation uint8, dbname string) (IContext, error) + // OpenCtx opens an IContext with connection id, client capability, collation and dbname. + OpenCtx(connID uint64, capability uint32, collation uint8, dbname string) (IContext, error) } // IContext is the interface to execute commant. diff --git a/tidb-server/server/driver_tidb.go b/tidb-server/server/driver_tidb.go index c7f6fce4d6..aa59f6f189 100644 --- a/tidb-server/server/driver_tidb.go +++ b/tidb-server/server/driver_tidb.go @@ -110,9 +110,10 @@ func (ts *TiDBStatement) Close() error { } // OpenCtx implements IDriver. -func (qd *TiDBDriver) OpenCtx(capability uint32, collation uint8, dbname string) (IContext, error) { +func (qd *TiDBDriver) OpenCtx(connID uint64, capability uint32, collation uint8, dbname string) (IContext, error) { session, _ := tidb.CreateSession(qd.store) session.SetClientCapability(capability) + session.SetConnectionID(connID) if dbname != "" { _, err := session.Execute("use " + dbname) if err != nil { From 6e1094c16a453d56ab0b2e78882e6739af652c7d Mon Sep 17 00:00:00 2001 From: shenli Date: Mon, 21 Dec 2015 23:27:45 +0800 Subject: [PATCH 49/63] parser: "variables" is an unreserved keyword --- parser/parser.y | 2 +- parser/parser_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parser/parser.y b/parser/parser.y index d1e7d4b891..dfc22da075 100644 --- a/parser/parser.y +++ b/parser/parser.y @@ -1684,7 +1684,7 @@ UnReservedKeyword: | "VALUE" | "WARNINGS" | "YEAR" | "MODE" | "WEEK" | "ANY" | "SOME" | "USER" | "IDENTIFIED" | "COLLATION" | "COMMENT" | "AVG_ROW_LENGTH" | "CONNECTION" | "CHECKSUM" | "COMPRESSION" | "KEY_BLOCK_SIZE" | "MAX_ROWS" | "MIN_ROWS" | "NATIONAL" | "ROW" | "QUARTER" | "ESCAPE" | "GRANTS" | "FIELDS" | "TRIGGERS" | "DELAY_KEY_WRITE" | "ISOLATION" -| "REPEATABLE" | "COMMITTED" | "UNCOMMITTED" | "ONLY" | "SERIALIZABLE" | "LEVEL" +| "REPEATABLE" | "COMMITTED" | "UNCOMMITTED" | "ONLY" | "SERIALIZABLE" | "LEVEL" | "VARIABLES" NotKeywordToken: "ABS" | "ADDDATE" | "COALESCE" | "CONCAT" | "CONCAT_WS" | "COUNT" | "DAY" | "DATE_ADD" | "DATE_SUB" | "DAYOFMONTH" diff --git a/parser/parser_test.go b/parser/parser_test.go index 57da97e210..e4ad34960c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -41,7 +41,7 @@ func (s *testParserSuite) TestSimple(c *C) { "collation", "comment", "avg_row_length", "checksum", "compression", "connection", "key_block_size", "max_rows", "min_rows", "national", "row", "quarter", "escape", "grants", "status", "fields", "triggers", "delay_key_write", "isolation", "repeatable", "committed", "uncommitted", "only", "serializable", "level", - "curtime", + "curtime", "variables", } for _, kw := range unreservedKws { src := fmt.Sprintf("SELECT %s FROM tbl;", kw) From 25d883da3ce2aac5371044cebb2bbb99bdd163bf Mon Sep 17 00:00:00 2001 From: shenli Date: Tue, 22 Dec 2015 00:17:12 +0800 Subject: [PATCH 50/63] *: Address comment --- executor/converter/convert_expr.go | 3 ++- session_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/executor/converter/convert_expr.go b/executor/converter/convert_expr.go index 1206952707..9baa1fba67 100644 --- a/executor/converter/convert_expr.go +++ b/executor/converter/convert_expr.go @@ -14,12 +14,13 @@ package converter import ( + "strings" + "github.com/juju/errors" "github.com/pingcap/tidb/ast" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/subquery" "github.com/pingcap/tidb/model" - "strings" ) func convertExpr(converter *expressionConverter, expr ast.ExprNode) (expression.Expression, error) { diff --git a/session_test.go b/session_test.go index 18ad9f5ffd..b7d597b22b 100644 --- a/session_test.go +++ b/session_test.go @@ -1373,8 +1373,8 @@ func (s *testSessionSuite) TestSubstringIndexExpr(c *C) { se := newSession(c, store, s.dbName) mustExecSQL(c, se, "drop table if exists t;") mustExecSQL(c, se, "create table t (c varchar(128));") - mustExecSQL(c, se, `insert into t values ("www.mysql.com");`) - mustExecMatch(c, se, "SELECT DISTINCT SUBSTRING_INDEX(c, '.', 2) from t;", [][]interface{}{{"www.mysql"}}) + mustExecSQL(c, se, `insert into t values ("www.pingcap.com");`) + mustExecMatch(c, se, "SELECT DISTINCT SUBSTRING_INDEX(c, '.', 2) from t;", [][]interface{}{{"www.pingcap"}}) } func (s *testSessionSuite) TestGlobalVarAccessor(c *C) { From eae212ba73a9cdb199b51f15e229f1ecbed8579b Mon Sep 17 00:00:00 2001 From: shenli Date: Tue, 22 Dec 2015 09:01:52 +0800 Subject: [PATCH 51/63] converter: Address comment --- executor/converter/convert_expr.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/executor/converter/convert_expr.go b/executor/converter/convert_expr.go index 9baa1fba67..4cc5bb799b 100644 --- a/executor/converter/convert_expr.go +++ b/executor/converter/convert_expr.go @@ -17,6 +17,7 @@ import ( "strings" "github.com/juju/errors" + "github.com/ngaut/log" "github.com/pingcap/tidb/ast" "github.com/pingcap/tidb/expression" "github.com/pingcap/tidb/expression/subquery" @@ -117,6 +118,8 @@ func (c *expressionConverter) Leave(in ast.Node) (out ast.Node, ok bool) { c.funcDateArith(v) case *ast.AggregateFuncExpr: c.aggregateFunc(v) + case ast.ExprNode: + log.Errorf("Unknown expr node %T", v) } return in, c.err == nil } From 889359b62c9d36ef749325bd779ad2a1dbd2933e Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Tue, 22 Dec 2015 09:22:52 +0800 Subject: [PATCH 52/63] kv: check error first in test. --- kv/index_iter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kv/index_iter_test.go b/kv/index_iter_test.go index 4d92eb2ff8..8817991b1c 100644 --- a/kv/index_iter_test.go +++ b/kv/index_iter_test.go @@ -156,8 +156,8 @@ func (s *testIndexSuite) TestCombineIndexSeek(c *C) { index2 := kv.NewKVIndex("i", "test", 1, false) iter, hit, err := index2.Seek(txn, []interface{}{"abc", nil}) - defer iter.Close() c.Assert(err, IsNil) + defer iter.Close() c.Assert(hit, IsFalse) _, h, err := iter.Next() c.Assert(err, IsNil) From 55c0e14ab41cdc587a36913221f37396e3bcc840 Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 09:39:19 +0800 Subject: [PATCH 53/63] Address comments --- store/localstore/boltdb/boltdb.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store/localstore/boltdb/boltdb.go b/store/localstore/boltdb/boltdb.go index ad1ec4de1c..4095f9c2fe 100644 --- a/store/localstore/boltdb/boltdb.go +++ b/store/localstore/boltdb/boltdb.go @@ -65,10 +65,12 @@ func (d *db) MultiSeek(keys [][]byte) []*engine.MSeekResult { } r := &engine.MSeekResult{} - r.Key, r.Value, r.Err = bytes.CloneBytes(k), bytes.CloneBytes(v), nil if k == nil { r.Err = engine.ErrNotFound + } else { + r.Key, r.Value, r.Err = bytes.CloneBytes(k), bytes.CloneBytes(v), nil } + res = append(res, r) } return nil From 462f6a971ed980ccbb8e29235ed50416ccc9340b Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 12:19:16 +0800 Subject: [PATCH 54/63] *: instroduce segment map --- store/localstore/kv.go | 11 +++--- util/segmentmap/segmentmap.go | 60 ++++++++++++++++++++++++++++++ util/segmentmap/segmentmap_test.go | 38 +++++++++++++++++++ 3 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 util/segmentmap/segmentmap.go create mode 100644 util/segmentmap/segmentmap_test.go diff --git a/store/localstore/kv.go b/store/localstore/kv.go index 8712acb8c9..13768a0d8b 100644 --- a/store/localstore/kv.go +++ b/store/localstore/kv.go @@ -21,6 +21,7 @@ import ( "github.com/ngaut/log" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/store/localstore/engine" + "github.com/pingcap/tidb/util/segmentmap" "github.com/twinj/uuid" ) @@ -168,12 +169,12 @@ func (s *dbStore) tryLock(txn *dbTxn) (err error) { return errors.Trace(kv.ErrLockConflict) } - lastVer, ok := s.recentUpdates[k] + lastVer, ok := s.recentUpdates.Get([]byte(k)) if !ok { continue } // If there's newer version of this key, returns error. - if lastVer.Cmp(kv.Version{Ver: txn.tid}) > 0 { + if lastVer.(kv.Version).Cmp(kv.Version{Ver: txn.tid}) > 0 { return errors.Trace(kv.ErrConditionNotMatch) } } @@ -243,7 +244,7 @@ type dbStore struct { txns map[uint64]*dbTxn keysLocked map[string]uint64 // TODO: clean up recentUpdates - recentUpdates map[string]kv.Version + recentUpdates *segmentmap.SegmentMap uuid string path string compactor *localstoreCompactor @@ -313,7 +314,7 @@ func (d Driver) Open(schema string) (kv.Storage, error) { commandCh: make(chan *command, 1000), closed: false, closeCh: make(chan struct{}), - recentUpdates: make(map[string]kv.Version), + recentUpdates: segmentmap.NewSegmentMap(100), wg: &sync.WaitGroup{}, } mc.cache[schema] = s @@ -428,7 +429,7 @@ func (s *dbStore) unLockKeys(txn *dbTxn) error { } delete(s.keysLocked, k) - s.recentUpdates[k] = txn.version + s.recentUpdates.Set([]byte(k), txn.version, true) } return nil diff --git a/util/segmentmap/segmentmap.go b/util/segmentmap/segmentmap.go new file mode 100644 index 0000000000..62407d70f9 --- /dev/null +++ b/util/segmentmap/segmentmap.go @@ -0,0 +1,60 @@ +package segmentmap + +import ( + "hash/crc32" + + "github.com/juju/errors" +) + +// SegmentMap is used for handle a big map slice by slice +type SegmentMap struct { + size int + maps []map[string]interface{} +} + +// NewSegmentMap crate a new SegmentMap +func NewSegmentMap(size int) *SegmentMap { + sm := &SegmentMap{ + maps: make([]map[string]interface{}, size), + size: size, + } + + for i := 0; i < size; i++ { + sm.maps[i] = make(map[string]interface{}) + } + return sm +} + +// Get is the same as map[k] +func (sm *SegmentMap) Get(key []byte) (interface{}, bool) { + idx := int(crc32.ChecksumIEEE(key)) % sm.size + val, ok := sm.maps[idx][string(key)] + return val, ok +} + +// GetSegment gets the map specific by index +func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { + if index >= len(sm.maps) { + return nil, errors.Errorf("index out of bound") + } + + return sm.maps[index], nil +} + +// Set if empty, return whether already exists +func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) (exist bool) { + idx := int(crc32.ChecksumIEEE(key)) % sm.size + k := string(key) + _, exist = sm.maps[idx][k] + if exist && !force { + return exist + } + + sm.maps[idx][k] = value + return exist +} + +// SegmentCount return how many inner segments +func (sm *SegmentMap) SegmentCount() int { + return sm.size +} diff --git a/util/segmentmap/segmentmap_test.go b/util/segmentmap/segmentmap_test.go new file mode 100644 index 0000000000..52768543e0 --- /dev/null +++ b/util/segmentmap/segmentmap_test.go @@ -0,0 +1,38 @@ +package segmentmap + +import ( + . "github.com/pingcap/check" +) + +var _ = Suite(&testSegmentMapSuite{}) + +type testSegmentMapSuite struct { +} + +func (s *testSegmentMapSuite) TestSegment(c *C) { + segs := 2 + m := NewSegmentMap(segs) + c.Assert(m.SegmentCount(), Equals, segs) + k := []byte("k") + v := []byte("v") + val, exist := m.Get(k) + c.Assert(exist, Equals, false) + + exist = m.Set(k, v, false) + c.Assert(exist, IsFalse) + + val, exist = m.Get(k) + c.Assert(v, DeepEquals, val.([]byte)) + c.Assert(exist, Equals, true) + + m0, err := m.GetSegment(0) + c.Assert(err, IsNil) + + m1, err := m.GetSegment(1) + c.Assert(err, IsNil) + + c.Assert(len(m0)+len(m1), Equals, 1) + + _, err = m.GetSegment(3) + c.Assert(err, NotNil) +} From 757358331d44e1f38aed29fcb97eb2e26caaa763 Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 12:41:20 +0800 Subject: [PATCH 55/63] Add license header --- util/segmentmap/segmentmap.go | 13 +++++++++++++ util/segmentmap/segmentmap_test.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/util/segmentmap/segmentmap.go b/util/segmentmap/segmentmap.go index 62407d70f9..1dc9f24f3d 100644 --- a/util/segmentmap/segmentmap.go +++ b/util/segmentmap/segmentmap.go @@ -1,3 +1,16 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package segmentmap import ( diff --git a/util/segmentmap/segmentmap_test.go b/util/segmentmap/segmentmap_test.go index 52768543e0..32c350d3fd 100644 --- a/util/segmentmap/segmentmap_test.go +++ b/util/segmentmap/segmentmap_test.go @@ -1,3 +1,16 @@ +// Copyright 2015 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package segmentmap import ( From 45bcc1d874d78f7e80eedebb44d58a5ca7a69ac9 Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 13:37:55 +0800 Subject: [PATCH 56/63] Clean up recent updates --- store/localstore/kv.go | 26 ++++++++++++++++++++++ store/localstore/local_version_provider.go | 10 ++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/store/localstore/kv.go b/store/localstore/kv.go index 13768a0d8b..d1e9f52b03 100644 --- a/store/localstore/kv.go +++ b/store/localstore/kv.go @@ -16,6 +16,7 @@ package localstore import ( "runtime/debug" "sync" + "time" "github.com/juju/errors" "github.com/ngaut/log" @@ -139,6 +140,10 @@ func (s *dbStore) scheduler() { go s.seekWorker(wgSeekWorkers, seekCh) } + segmentIndex := 0 + + tick := time.NewTicker(time.Second) + for { select { case cmd := <-s.commandCh: @@ -158,6 +163,27 @@ func (s *dbStore) scheduler() { close(seekCh) wgSeekWorkers.Wait() s.wg.Done() + case <-tick.C: + segmentIndex = segmentIndex % s.recentUpdates.SegmentCount() + s.cleanRecentUpdates(segmentIndex) + segmentIndex++ + } + } +} + +func (s *dbStore) cleanRecentUpdates(segmentIndex int) { + m, err := s.recentUpdates.GetSegment(segmentIndex) + if err != nil { + log.Error(err) + return + } + + lowerWaterMark := int64(10) // second + now := time.Now().UnixNano() / int64(time.Second) + for k, v := range m { + dis := now - version2Second(v.(kv.Version)) + if dis > lowerWaterMark { + delete(m, k) } } } diff --git a/store/localstore/local_version_provider.go b/store/localstore/local_version_provider.go index 8450647678..6a2cdb8cd4 100644 --- a/store/localstore/local_version_provider.go +++ b/store/localstore/local_version_provider.go @@ -27,6 +27,14 @@ const ( timePrecisionOffset = 18 ) +func time2TsPhysical(t time.Time) uint64 { + return uint64((t.UnixNano() / int64(time.Millisecond)) << timePrecisionOffset) +} + +func version2Second(v kv.Version) int64 { + return int64(v.Ver>>timePrecisionOffset) / 1000 +} + // CurrentVersion implements the VersionProvider's GetCurrentVer interface. func (l *LocalVersionProvider) CurrentVersion() (kv.Version, error) { l.mu.Lock() @@ -34,7 +42,7 @@ func (l *LocalVersionProvider) CurrentVersion() (kv.Version, error) { for { var ts uint64 - ts = uint64((time.Now().UnixNano() / int64(time.Millisecond)) << timePrecisionOffset) + ts = time2TsPhysical(time.Now()) if l.lastTimestamp > ts { log.Error("[kv] invalid physical time stamp") From b89f8c77ad55af83adfd2b26ec18df1106c08f0e Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Tue, 22 Dec 2015 13:46:54 +0800 Subject: [PATCH 57/63] *: remove goyacc/golexer/godep from make update. --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index a69e0a85e7..5df1c31318 100644 --- a/Makefile +++ b/Makefile @@ -55,12 +55,9 @@ install: $(GO) install ./... update: - go get -u github.com/tools/godep go get -u github.com/pingcap/go-hbase go get -u github.com/pingcap/go-themis go get -u github.com/ngaut/tso/client - go get -u github.com/qiuyesuifeng/goyacc - go get -u github.com/qiuyesuifeng/golex TEMP_FILE = temp_parser_file From f91303cfd44f14f812702b0a117fa340c856414a Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 13:50:57 +0800 Subject: [PATCH 58/63] Address comments --- util/segmentmap/segmentmap.go | 7 ++++--- util/segmentmap/segmentmap_test.go | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/util/segmentmap/segmentmap.go b/util/segmentmap/segmentmap.go index 1dc9f24f3d..7f1e31fdf9 100644 --- a/util/segmentmap/segmentmap.go +++ b/util/segmentmap/segmentmap.go @@ -20,6 +20,7 @@ import ( ) // SegmentMap is used for handle a big map slice by slice +// It's not thread safe type SegmentMap struct { size int maps []map[string]interface{} @@ -48,17 +49,17 @@ func (sm *SegmentMap) Get(key []byte) (interface{}, bool) { // GetSegment gets the map specific by index func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { if index >= len(sm.maps) { - return nil, errors.Errorf("index out of bound") + return nil, errors.Errorf("index out of bound: %d", index) } return sm.maps[index], nil } // Set if empty, return whether already exists -func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) (exist bool) { +func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool { idx := int(crc32.ChecksumIEEE(key)) % sm.size k := string(key) - _, exist = sm.maps[idx][k] + _, exist := sm.maps[idx][k] if exist && !force { return exist } diff --git a/util/segmentmap/segmentmap_test.go b/util/segmentmap/segmentmap_test.go index 32c350d3fd..b8610a5d90 100644 --- a/util/segmentmap/segmentmap_test.go +++ b/util/segmentmap/segmentmap_test.go @@ -29,14 +29,14 @@ func (s *testSegmentMapSuite) TestSegment(c *C) { k := []byte("k") v := []byte("v") val, exist := m.Get(k) - c.Assert(exist, Equals, false) + c.Assert(exist, IsFalse) exist = m.Set(k, v, false) c.Assert(exist, IsFalse) val, exist = m.Get(k) c.Assert(v, DeepEquals, val.([]byte)) - c.Assert(exist, Equals, true) + c.Assert(exist, IsTrue) m0, err := m.GetSegment(0) c.Assert(err, IsNil) From d9c02322e631d4eb9a65c9667af8ed6a2838f736 Mon Sep 17 00:00:00 2001 From: qiuyesuifeng Date: Tue, 22 Dec 2015 17:17:50 +0800 Subject: [PATCH 59/63] *: update check godep. --- Godeps/Godeps.json | 2 +- .../src/github.com/pingcap/check/checkers2.go | 5 +-- .../pingcap/check/checkers2_test.go | 9 ++++++ .../src/github.com/pingcap/check/compare.go | 31 ++++++++++++++----- .../pingcap/check/godropbox_license | 27 ++++++++++++++++ 5 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/pingcap/check/godropbox_license diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1ced2f3ca4..942344b977 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -37,7 +37,7 @@ }, { "ImportPath": "github.com/pingcap/check", - "Rev": "dbd36251e82f4d1aed5561647b402a448839c412" + "Rev": "ce8a2f822ab1e245a4eefcef2996531c79c943f1" }, { "ImportPath": "github.com/rcrowley/go-metrics", diff --git a/Godeps/_workspace/src/github.com/pingcap/check/checkers2.go b/Godeps/_workspace/src/github.com/pingcap/check/checkers2.go index 6535b8d53c..c09bcdc5eb 100644 --- a/Godeps/_workspace/src/github.com/pingcap/check/checkers2.go +++ b/Godeps/_workspace/src/github.com/pingcap/check/checkers2.go @@ -69,10 +69,7 @@ func (b *bytesEquals) Check(params []interface{}, names []string) (bool, string) return false, "Arguments to BytesEqual must both be bytestrings" } - if bytes.Equal(b1, b2) { - return true, "" - } - return false, "Byte arrays were different" + return bytes.Equal(b1, b2), "" } func (b *bytesEquals) Info() *CheckerInfo { diff --git a/Godeps/_workspace/src/github.com/pingcap/check/checkers2_test.go b/Godeps/_workspace/src/github.com/pingcap/check/checkers2_test.go index ffcd2e32b8..699ba47a62 100644 --- a/Godeps/_workspace/src/github.com/pingcap/check/checkers2_test.go +++ b/Godeps/_workspace/src/github.com/pingcap/check/checkers2_test.go @@ -1,6 +1,8 @@ package check_test import ( + "time" + "github.com/pingcap/check" ) @@ -34,4 +36,11 @@ func (s *CheckersS) TestCompare(c *check.C) { c.Assert("ABC", check.Less, "ABCD") c.Assert([]byte("ABC"), check.Less, []byte("ABCD")) c.Assert(3.14, check.Less, 3.145) + c.Assert(time.Duration(1), check.Greater, time.Duration(0)) + c.Assert(time.Now(), check.Less, time.Now().Add(10*time.Second)) +} + +func (s *CheckersS) TestBytes(c *check.C) { + c.Assert([]byte{0x00}, check.BytesEquals, []byte{0x00}) + c.Assert([]byte{0x00}, check.Not(check.BytesEquals), []byte{0x01}) } diff --git a/Godeps/_workspace/src/github.com/pingcap/check/compare.go b/Godeps/_workspace/src/github.com/pingcap/check/compare.go index 60041528b2..7005cba704 100644 --- a/Godeps/_workspace/src/github.com/pingcap/check/compare.go +++ b/Godeps/_workspace/src/github.com/pingcap/check/compare.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "reflect" + "time" ) type compareFunc func(v1 interface{}, v2 interface{}) (bool, error) @@ -33,9 +34,8 @@ func compare(v1 interface{}, v2 interface{}) (int, error) { return 1, nil } else if a1 == a2 { return 0, nil - } else { - return -1, nil } + return -1, nil case uint, uint8, uint16, uint32, uint64: a1 := value1.Uint() a2 := value2.Uint() @@ -43,9 +43,8 @@ func compare(v1 interface{}, v2 interface{}) (int, error) { return 1, nil } else if a1 == a2 { return 0, nil - } else { - return -1, nil } + return -1, nil case float32, float64: a1 := value1.Float() a2 := value2.Float() @@ -53,9 +52,8 @@ func compare(v1 interface{}, v2 interface{}) (int, error) { return 1, nil } else if a1 == a2 { return 0, nil - } else { - return -1, nil } + return -1, nil case string: a1 := value1.String() a2 := value2.String() @@ -63,13 +61,30 @@ func compare(v1 interface{}, v2 interface{}) (int, error) { return 1, nil } else if a1 == a2 { return 0, nil - } else { - return -1, nil } + return -1, nil case []byte: a1 := value1.Bytes() a2 := value2.Bytes() return bytes.Compare(a1, a2), nil + case time.Time: + a1 := v1.(time.Time) + a2 := v2.(time.Time) + if a1.After(a2) { + return 1, nil + } else if a1.Equal(a2) { + return 0, nil + } + return -1, nil + case time.Duration: + a1 := v1.(time.Duration) + a2 := v2.(time.Duration) + if a1 > a2 { + return 1, nil + } else if a1 == a2 { + return 0, nil + } + return -1, nil default: return 0, fmt.Errorf("type %T is not supported now", v1) } diff --git a/Godeps/_workspace/src/github.com/pingcap/check/godropbox_license b/Godeps/_workspace/src/github.com/pingcap/check/godropbox_license new file mode 100644 index 0000000000..04d42108ee --- /dev/null +++ b/Godeps/_workspace/src/github.com/pingcap/check/godropbox_license @@ -0,0 +1,27 @@ +Copyright (c) 2014 Dropbox, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 28c89dfc0b1b89d7fe6ce44445681e2a76f20997 Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 18:57:52 +0800 Subject: [PATCH 60/63] Use Castagnoli to calculate hash --- util/segmentmap/segmentmap.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/util/segmentmap/segmentmap.go b/util/segmentmap/segmentmap.go index 7f1e31fdf9..229be131ee 100644 --- a/util/segmentmap/segmentmap.go +++ b/util/segmentmap/segmentmap.go @@ -24,6 +24,8 @@ import ( type SegmentMap struct { size int maps []map[string]interface{} + + crcTable *crc32.Table } // NewSegmentMap crate a new SegmentMap @@ -36,19 +38,21 @@ func NewSegmentMap(size int) *SegmentMap { for i := 0; i < size; i++ { sm.maps[i] = make(map[string]interface{}) } + + sm.crcTable = crc32.MakeTable(crc32.Castagnoli) return sm } // Get is the same as map[k] func (sm *SegmentMap) Get(key []byte) (interface{}, bool) { - idx := int(crc32.ChecksumIEEE(key)) % sm.size + idx := int(crc32.Checksum(key, sm.crcTable)) % sm.size val, ok := sm.maps[idx][string(key)] return val, ok } // GetSegment gets the map specific by index func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { - if index >= len(sm.maps) { + if index >= sm.size || index < 0 { return nil, errors.Errorf("index out of bound: %d", index) } @@ -57,7 +61,7 @@ func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { // Set if empty, return whether already exists func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool { - idx := int(crc32.ChecksumIEEE(key)) % sm.size + idx := int(crc32.Checksum(key, sm.crcTable)) % sm.size k := string(key) _, exist := sm.maps[idx][k] if exist && !force { From 77201c39b346286a020973b403c7f4780190bb58 Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 19:59:27 +0800 Subject: [PATCH 61/63] Address comments --- store/localstore/kv.go | 28 ++++++++++++++++------------ util/segmentmap/segmentmap.go | 19 +++++++++++-------- util/segmentmap/segmentmap_test.go | 3 ++- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/store/localstore/kv.go b/store/localstore/kv.go index d1e9f52b03..8599788c31 100644 --- a/store/localstore/kv.go +++ b/store/localstore/kv.go @@ -143,6 +143,7 @@ func (s *dbStore) scheduler() { segmentIndex := 0 tick := time.NewTicker(time.Second) + defer tick.Stop() for { select { @@ -179,7 +180,7 @@ func (s *dbStore) cleanRecentUpdates(segmentIndex int) { } lowerWaterMark := int64(10) // second - now := time.Now().UnixNano() / int64(time.Second) + now := time.Now().Unix() for k, v := range m { dis := now - version2Second(v.(kv.Version)) if dis > lowerWaterMark { @@ -331,17 +332,20 @@ func (d Driver) Open(schema string) (kv.Storage, error) { log.Info("[kv] New store", schema) s := &dbStore{ - txns: make(map[uint64]*dbTxn), - keysLocked: make(map[string]uint64), - uuid: uuid.NewV4().String(), - path: schema, - db: db, - compactor: newLocalCompactor(localCompactDefaultPolicy, db), - commandCh: make(chan *command, 1000), - closed: false, - closeCh: make(chan struct{}), - recentUpdates: segmentmap.NewSegmentMap(100), - wg: &sync.WaitGroup{}, + txns: make(map[uint64]*dbTxn), + keysLocked: make(map[string]uint64), + uuid: uuid.NewV4().String(), + path: schema, + db: db, + compactor: newLocalCompactor(localCompactDefaultPolicy, db), + commandCh: make(chan *command, 1000), + closed: false, + closeCh: make(chan struct{}), + wg: &sync.WaitGroup{}, + } + s.recentUpdates, err = segmentmap.NewSegmentMap(100) + if err != nil { + return nil, errors.Trace(err) } mc.cache[schema] = s s.compactor.Start() diff --git a/util/segmentmap/segmentmap.go b/util/segmentmap/segmentmap.go index 229be131ee..554ef36ecc 100644 --- a/util/segmentmap/segmentmap.go +++ b/util/segmentmap/segmentmap.go @@ -20,7 +20,7 @@ import ( ) // SegmentMap is used for handle a big map slice by slice -// It's not thread safe +// It's not thread safe. type SegmentMap struct { size int maps []map[string]interface{} @@ -28,8 +28,11 @@ type SegmentMap struct { crcTable *crc32.Table } -// NewSegmentMap crate a new SegmentMap -func NewSegmentMap(size int) *SegmentMap { +// NewSegmentMap create a new SegmentMap. +func NewSegmentMap(size int) (*SegmentMap, error) { + if size <= 0 { + return nil, errors.Errorf("Invalid size: %d", size) + } sm := &SegmentMap{ maps: make([]map[string]interface{}, size), size: size, @@ -40,17 +43,17 @@ func NewSegmentMap(size int) *SegmentMap { } sm.crcTable = crc32.MakeTable(crc32.Castagnoli) - return sm + return sm, nil } -// Get is the same as map[k] +// Get is the same as map[k]. func (sm *SegmentMap) Get(key []byte) (interface{}, bool) { idx := int(crc32.Checksum(key, sm.crcTable)) % sm.size val, ok := sm.maps[idx][string(key)] return val, ok } -// GetSegment gets the map specific by index +// GetSegment gets the map specific by index. func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { if index >= sm.size || index < 0 { return nil, errors.Errorf("index out of bound: %d", index) @@ -59,7 +62,7 @@ func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { return sm.maps[index], nil } -// Set if empty, return whether already exists +// Set if empty, returns whether already exists. func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool { idx := int(crc32.Checksum(key, sm.crcTable)) % sm.size k := string(key) @@ -72,7 +75,7 @@ func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool { return exist } -// SegmentCount return how many inner segments +// SegmentCount returns how many inner segments. func (sm *SegmentMap) SegmentCount() int { return sm.size } diff --git a/util/segmentmap/segmentmap_test.go b/util/segmentmap/segmentmap_test.go index b8610a5d90..2a86786999 100644 --- a/util/segmentmap/segmentmap_test.go +++ b/util/segmentmap/segmentmap_test.go @@ -24,7 +24,8 @@ type testSegmentMapSuite struct { func (s *testSegmentMapSuite) TestSegment(c *C) { segs := 2 - m := NewSegmentMap(segs) + m, err := NewSegmentMap(segs) + c.Assert(err, IsNil) c.Assert(m.SegmentCount(), Equals, segs) k := []byte("k") v := []byte("v") From 3dc9261d816a91c6cd62c635e5bbf1d930b94994 Mon Sep 17 00:00:00 2001 From: ngaut Date: Tue, 22 Dec 2015 20:20:08 +0800 Subject: [PATCH 62/63] Address comments --- store/localstore/kv.go | 7 ++----- util/segmentmap/segmentmap.go | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/store/localstore/kv.go b/store/localstore/kv.go index 8599788c31..41e0c30c3b 100644 --- a/store/localstore/kv.go +++ b/store/localstore/kv.go @@ -39,6 +39,8 @@ const ( const ( maxSeekWorkers = 3 + + lowerWaterMark = 10 // second ) type command struct { @@ -65,8 +67,6 @@ type seekArgs struct { type commitArgs struct { } -//scheduler - // Seek searches for the first key in the engine which is >= key in byte order, returns (nil, nil, ErrNotFound) // if such key is not found. func (s *dbStore) Seek(key []byte) ([]byte, []byte, error) { @@ -179,7 +179,6 @@ func (s *dbStore) cleanRecentUpdates(segmentIndex int) { return } - lowerWaterMark := int64(10) // second now := time.Now().Unix() for k, v := range m { dis := now - version2Second(v.(kv.Version)) @@ -263,8 +262,6 @@ func (s *dbStore) NewBatch() engine.Batch { return s.db.NewBatch() } -//end of scheduler - type dbStore struct { db engine.DB diff --git a/util/segmentmap/segmentmap.go b/util/segmentmap/segmentmap.go index 554ef36ecc..11edbab382 100644 --- a/util/segmentmap/segmentmap.go +++ b/util/segmentmap/segmentmap.go @@ -19,7 +19,7 @@ import ( "github.com/juju/errors" ) -// SegmentMap is used for handle a big map slice by slice +// SegmentMap is used for handle a big map slice by slice. // It's not thread safe. type SegmentMap struct { size int @@ -33,11 +33,11 @@ func NewSegmentMap(size int) (*SegmentMap, error) { if size <= 0 { return nil, errors.Errorf("Invalid size: %d", size) } + sm := &SegmentMap{ maps: make([]map[string]interface{}, size), size: size, } - for i := 0; i < size; i++ { sm.maps[i] = make(map[string]interface{}) } @@ -62,7 +62,7 @@ func (sm *SegmentMap) GetSegment(index int) (map[string]interface{}, error) { return sm.maps[index], nil } -// Set if empty, returns whether already exists. +// Set if key not exists, returns whether already exists. func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool { idx := int(crc32.Checksum(key, sm.crcTable)) % sm.size k := string(key) From 7130e4bde71016e4e27e775830ddf1c73db9d1fa Mon Sep 17 00:00:00 2001 From: Ewan Chou Date: Tue, 22 Dec 2015 21:38:10 +0800 Subject: [PATCH 63/63] tide-server: avoid panic when failed in SetupSuite. Have taken a look at `go check` source code, `TearDownSuite` will always be called even if it failed in `SetUpSuite`. --- tidb-server/server/tidb_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tidb-server/server/tidb_test.go b/tidb-server/server/tidb_test.go index e60ec6d025..adb0a2f371 100644 --- a/tidb-server/server/tidb_test.go +++ b/tidb-server/server/tidb_test.go @@ -43,7 +43,9 @@ func (ts *TidbTestSuite) SetUpSuite(c *C) { } func (ts *TidbTestSuite) TearDownSuite(c *C) { - ts.server.Close() + if ts.server != nil { + ts.server.Close() + } } func (ts *TidbTestSuite) TestRegression(c *C) {