From ef71bb7e2a97c87d2cb982f190a78b0e098713eb Mon Sep 17 00:00:00 2001 From: disksing Date: Thu, 10 Dec 2015 17:14:50 +0800 Subject: [PATCH] util/codec: address comments --- kv/index_iter.go | 2 +- table/tables/tables.go | 2 +- util/codec/bytes.go | 9 +- util/codec/codec.go | 203 ++++++++++----------------------------- util/codec/codec_test.go | 10 +- util/codec/number.go | 2 + 6 files changed, 64 insertions(+), 164 deletions(-) diff --git a/kv/index_iter.go b/kv/index_iter.go index 7cad089db9..a08c007c08 100644 --- a/kv/index_iter.go +++ b/kv/index_iter.go @@ -69,7 +69,7 @@ func (c *indexIter) Next() (k []interface{}, h int64, err error) { } // get indexedValues buf := []byte(c.it.Key())[len(c.prefix):] - vv, err := codec.DecodeKey(buf) + vv, err := codec.Decode(buf) if err != nil { return nil, 0, errors.Trace(err) } diff --git a/table/tables/tables.go b/table/tables/tables.go index c6553603bb..b96ad8527c 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -495,7 +495,7 @@ func (t *Table) EncodeValue(raw interface{}) ([]byte, error) { // DecodeValue implements table.Table DecodeValue interface. func (t *Table) DecodeValue(data []byte, col *column.Col) (interface{}, error) { - values, err := codec.DecodeValue(data) + values, err := codec.Decode(data) if err != nil { return nil, errors.Trace(err) } diff --git a/util/codec/bytes.go b/util/codec/bytes.go index 4ce078cf8b..aa023b95d0 100644 --- a/util/codec/bytes.go +++ b/util/codec/bytes.go @@ -15,6 +15,7 @@ package codec import ( "bytes" + "encoding/binary" "runtime" "unsafe" @@ -123,8 +124,10 @@ func DecodeBytesDesc(b []byte) ([]byte, []byte, error) { } // EncodeCompactBytes joins bytes with its length into a byte slice. It is more -// efficient in both space and time compare to EncodeBytes. +// efficient in both space and time compare to EncodeBytes. Note that the encoded +// result is not memcomparable. func EncodeCompactBytes(b []byte, data []byte) []byte { + b = reallocBytes(b, binary.MaxVarintLen64+len(data)) b = EncodeVarint(b, int64(len(data))) return append(b, data...) } @@ -138,9 +141,7 @@ func DecodeCompactBytes(b []byte) ([]byte, []byte, error) { if int64(len(b)) < n { return nil, nil, errors.Errorf("insufficient bytes to decode value, expected length: %v", n) } - data := make([]byte, int(n)) - copy(data, b[:n]) - return b[n:], data, nil + return b[n:], b[:n], nil } // See https://golang.org/src/crypto/cipher/xor.go diff --git a/util/codec/codec.go b/util/codec/codec.go index 15c28a0841..95131b8b39 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -23,7 +23,9 @@ import ( const ( nilFlag byte = iota bytesFlag + compactBytesFlag stringFlag + compactStringFlag intFlag uintFlag floatFlag @@ -31,9 +33,7 @@ const ( durationFlag ) -// EncodeKey encodes args to a slice which can be sorted lexicographically later. -// EncodeKey guarantees the encoded slice is in ascending order for comparison. -func EncodeKey(args ...interface{}) ([]byte, error) { +func encode(args []interface{}, comparable bool) ([]byte, error) { var b []byte for _, arg := range args { switch v := arg.(type) { @@ -81,14 +81,32 @@ func EncodeKey(args ...interface{}) ([]byte, error) { b = append(b, floatFlag) b = EncodeFloat(b, float64(v)) case string: - b = append(b, stringFlag) - b = EncodeBytes(b, []byte(v)) + if comparable { + b = append(b, stringFlag) + b = EncodeBytes(b, []byte(v)) + } else { + b = append(b, compactStringFlag) + b = EncodeCompactBytes(b, []byte(v)) + } + case []byte: - b = append(b, bytesFlag) - b = EncodeBytes(b, v) + if comparable { + b = append(b, bytesFlag) + b = EncodeBytes(b, v) + } else { + b = append(b, compactBytesFlag) + b = EncodeCompactBytes(b, v) + } + case mysql.Time: - b = append(b, stringFlag) - b = EncodeBytes(b, []byte(v.String())) + if comparable { + b = append(b, stringFlag) + b = EncodeBytes(b, []byte(v.String())) + } else { + b = append(b, compactStringFlag) + b = EncodeCompactBytes(b, []byte(v.String())) + } + case mysql.Duration: // duration may have negative value, so we cannot use String to encode directly. b = append(b, durationFlag) @@ -118,8 +136,21 @@ func EncodeKey(args ...interface{}) ([]byte, error) { return b, nil } -// DecodeKey decodes values from a byte slice generated with EncodeKey before. -func DecodeKey(b []byte) ([]interface{}, error) { +// EncodeKey encodes args to a slice which can be sorted lexicographically later. +// EncodeKey guarantees the encoded slice is in ascending order for comparison. +func EncodeKey(args ...interface{}) ([]byte, error) { + return encode(args, true) +} + +// EncodeValue encodes args to a byte slice which can be decoded later. +// It does not guarantee the order for comparison. +func EncodeValue(args ...interface{}) ([]byte, error) { + return encode(args, false) +} + +// Decode decodes values from a byte slice generated with EncodeKey or EncodeValue +// before. +func Decode(b []byte) ([]interface{}, error) { if len(b) < 1 { return nil, errors.New("invalid encoded key") } @@ -143,12 +174,20 @@ func DecodeKey(b []byte) ([]interface{}, error) { b, v, err = DecodeFloat(b) case bytesFlag: b, v, err = DecodeBytes(b) + case compactBytesFlag: + b, v, err = DecodeCompactBytes(b) case stringFlag: var r []byte b, r, err = DecodeBytes(b) if err == nil { v = string(r) } + case compactStringFlag: + var r []byte + b, r, err = DecodeCompactBytes(b) + if err == nil { + v = string(r) + } case decimalFlag: b, v, err = DecodeDecimal(b) case durationFlag: @@ -172,145 +211,3 @@ func DecodeKey(b []byte) ([]interface{}, error) { return values, nil } - -// EncodeValue encodes args to a byte slice which can be decoded by DecodeValue later. -// It does not guarantee the order for comparison. -func EncodeValue(args ...interface{}) ([]byte, error) { - var b []byte - for _, arg := range args { - switch v := arg.(type) { - case bool: - b = append(b, intFlag) - if v { - b = EncodeInt(b, int64(1)) - } else { - b = EncodeInt(b, int64(0)) - } - case int: - b = append(b, intFlag) - b = EncodeInt(b, int64(v)) - case int8: - b = append(b, intFlag) - b = EncodeInt(b, int64(v)) - case int16: - b = append(b, intFlag) - b = EncodeInt(b, int64(v)) - case int32: - b = append(b, intFlag) - b = EncodeInt(b, int64(v)) - case int64: - b = append(b, intFlag) - b = EncodeInt(b, int64(v)) - case uint: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v)) - case uint8: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v)) - case uint16: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v)) - case uint32: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v)) - case uint64: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v)) - case float32: - b = append(b, floatFlag) - b = EncodeFloat(b, float64(v)) - case float64: - b = append(b, floatFlag) - b = EncodeFloat(b, float64(v)) - case string: - b = append(b, stringFlag) - b = EncodeCompactBytes(b, []byte(v)) - case []byte: - b = append(b, bytesFlag) - b = EncodeCompactBytes(b, v) - case mysql.Time: - b = append(b, stringFlag) - b = EncodeCompactBytes(b, []byte(v.String())) - case mysql.Duration: - // duration may have negative value, so we cannot use String to encode directly. - b = append(b, durationFlag) - b = EncodeInt(b, int64(v.Duration)) - case mysql.Decimal: - b = append(b, decimalFlag) - b = EncodeDecimal(b, v) - case mysql.Hex: - b = append(b, intFlag) - b = EncodeInt(b, int64(v.ToNumber())) - case mysql.Bit: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v.ToNumber())) - case mysql.Enum: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v.ToNumber())) - case mysql.Set: - b = append(b, uintFlag) - b = EncodeUint(b, uint64(v.ToNumber())) - case nil: - b = append(b, nilFlag) - default: - return nil, errors.Errorf("unsupport encode type %T", arg) - } - } - - return b, nil -} - -// DecodeValue decodes values from a byte slice generated with EncodeValue before. -func DecodeValue(b []byte) ([]interface{}, error) { - if len(b) < 1 { - return nil, errors.New("invalid encoded value") - } - - var ( - flag byte - err error - v interface{} - values = make([]interface{}, 0) - ) - - for len(b) > 0 { - flag = b[0] - b = b[1:] - switch flag { - case intFlag: - b, v, err = DecodeInt(b) - case uintFlag: - b, v, err = DecodeUint(b) - case floatFlag: - b, v, err = DecodeFloat(b) - case bytesFlag: - b, v, err = DecodeCompactBytes(b) - case stringFlag: - var r []byte - b, r, err = DecodeCompactBytes(b) - if err == nil { - v = string(r) - } - case decimalFlag: - b, v, err = DecodeDecimal(b) - case durationFlag: - var r int64 - b, r, err = DecodeInt(b) - if err == nil { - // use max fsp, let outer to do round manually. - v = mysql.Duration{Duration: time.Duration(r), Fsp: mysql.MaxFsp} - } - case nilFlag: - v = nil - default: - return nil, errors.Errorf("invalid encoded value flag %v", flag) - } - if err != nil { - return nil, errors.Trace(err) - } - - values = append(values, v) - } - - return values, nil -} diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index 5427ea24a3..7009b1c699 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -85,13 +85,13 @@ func (s *testCodecSuite) TestCodecKey(c *C) { for _, t := range table { b, err := EncodeKey(t.Input...) c.Assert(err, IsNil) - args, err := DecodeKey(b) + args, err := Decode(b) c.Assert(err, IsNil) c.Assert(args, DeepEquals, t.Expect) b, err = EncodeValue(t.Input...) c.Assert(err, IsNil) - args, err = DecodeValue(b) + args, err = Decode(b) c.Assert(err, IsNil) c.Assert(args, DeepEquals, t.Expect) } @@ -493,7 +493,7 @@ func (s *testCodecSuite) TestTime(c *C) { b, err := EncodeKey(m) c.Assert(err, IsNil) - v, err := DecodeKey(b) + v, err := Decode(b) c.Assert(err, IsNil) c.Assert(v, DeepEquals, []interface{}{t}) } @@ -534,7 +534,7 @@ func (s *testCodecSuite) TestDuration(c *C) { b, err := EncodeKey(m) c.Assert(err, IsNil) - v, err := DecodeKey(b) + v, err := Decode(b) c.Assert(err, IsNil) m.Fsp = mysql.MaxFsp c.Assert(v, DeepEquals, []interface{}{m}) @@ -586,7 +586,7 @@ func (s *testCodecSuite) TestDecimal(c *C) { c.Assert(err, IsNil) b, err := EncodeKey(m) c.Assert(err, IsNil) - v, err := DecodeKey(b) + v, err := Decode(b) c.Assert(err, IsNil) c.Assert(v, HasLen, 1) vv, ok := v[0].(mysql.Decimal) diff --git a/util/codec/number.go b/util/codec/number.go index 8a2f5fcbdc..cad729c31c 100644 --- a/util/codec/number.go +++ b/util/codec/number.go @@ -128,6 +128,7 @@ func DecodeUintDesc(b []byte) ([]byte, uint64, error) { } // EncodeVarint appends the encoded value to slice b and returns the appended slice. +// Note that the encoded result is not memcomparable. func EncodeVarint(b []byte, v int64) []byte { var data [binary.MaxVarintLen64]byte n := binary.PutVarint(data[:], v) @@ -148,6 +149,7 @@ func DecodeVarint(b []byte) ([]byte, int64, error) { } // EncodeUvarint appends the encoded value to slice b and returns the appended slice. +// Note that the encoded result is not memcomparable. func EncodeUvarint(b []byte, v uint64) []byte { var data [binary.MaxVarintLen64]byte n := binary.PutUvarint(data[:], v)