database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

util/codec: address comments

Browse Source
This commit is contained in:
disksing
2015-12-10 17:14:50 +08:00
parent 9b4f43e62f
commit ef71bb7e2a
6 changed files with 64 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kv/index_iter.go
View File

@ -69,7 +69,7 @@ func (c *indexIter) Next() (k []interface{}, h int64, err error) {
}
// get indexedValues
buf := []byte(c.it.Key())[len(c.prefix):]
vv, err := codec.DecodeKey(buf)
vv, err := codec.Decode(buf)
if err != nil {
return nil, 0, errors.Trace(err)
}

2
table/tables/tables.go
View File

@ -495,7 +495,7 @@ func (t *Table) EncodeValue(raw interface{}) ([]byte, error) {
// DecodeValue implements table.Table DecodeValue interface.
func (t *Table) DecodeValue(data []byte, col *column.Col) (interface{}, error) {
values, err := codec.DecodeValue(data)
values, err := codec.Decode(data)
if err != nil {
return nil, errors.Trace(err)
}

9
util/codec/bytes.go
View File

@ -15,6 +15,7 @@ package codec
import (
"bytes"
"encoding/binary"
"runtime"
"unsafe"
@ -123,8 +124,10 @@ func DecodeBytesDesc(b []byte) ([]byte, []byte, error) {
}
// EncodeCompactBytes joins bytes with its length into a byte slice. It is more
// efficient in both space and time compare to EncodeBytes.
// efficient in both space and time compare to EncodeBytes. Note that the encoded
// result is not memcomparable.
func EncodeCompactBytes(b []byte, data []byte) []byte {
b = reallocBytes(b, binary.MaxVarintLen64+len(data))
b = EncodeVarint(b, int64(len(data)))
return append(b, data...)
}
@ -138,9 +141,7 @@ func DecodeCompactBytes(b []byte) ([]byte, []byte, error) {
if int64(len(b)) < n {
return nil, nil, errors.Errorf("insufficient bytes to decode value, expected length: %v", n)
}
data := make([]byte, int(n))
copy(data, b[:n])
return b[n:], data, nil
return b[n:], b[:n], nil
}
// See https://golang.org/src/crypto/cipher/xor.go

203
util/codec/codec.go
View File

@ -23,7 +23,9 @@ import (
const (
nilFlag byte = iota
bytesFlag
compactBytesFlag
stringFlag
compactStringFlag
intFlag
uintFlag
floatFlag
@ -31,9 +33,7 @@ const (
durationFlag
)
// EncodeKey encodes args to a slice which can be sorted lexicographically later.
// EncodeKey guarantees the encoded slice is in ascending order for comparison.
func EncodeKey(args ...interface{}) ([]byte, error) {
func encode(args []interface{}, comparable bool) ([]byte, error) {
var b []byte
for _, arg := range args {
switch v := arg.(type) {
@ -81,14 +81,32 @@ func EncodeKey(args ...interface{}) ([]byte, error) {
b = append(b, floatFlag)
b = EncodeFloat(b, float64(v))
case string:
b = append(b, stringFlag)
b = EncodeBytes(b, []byte(v))
if comparable {
b = append(b, stringFlag)
b = EncodeBytes(b, []byte(v))
} else {
b = append(b, compactStringFlag)
b = EncodeCompactBytes(b, []byte(v))
}
case []byte:
b = append(b, bytesFlag)
b = EncodeBytes(b, v)
if comparable {
b = append(b, bytesFlag)
b = EncodeBytes(b, v)
} else {
b = append(b, compactBytesFlag)
b = EncodeCompactBytes(b, v)
}
case mysql.Time:
b = append(b, stringFlag)
b = EncodeBytes(b, []byte(v.String()))
if comparable {
b = append(b, stringFlag)
b = EncodeBytes(b, []byte(v.String()))
} else {
b = append(b, compactStringFlag)
b = EncodeCompactBytes(b, []byte(v.String()))
}
case mysql.Duration:
// duration may have negative value, so we cannot use String to encode directly.
b = append(b, durationFlag)
@ -118,8 +136,21 @@ func EncodeKey(args ...interface{}) ([]byte, error) {
return b, nil
}
// DecodeKey decodes values from a byte slice generated with EncodeKey before.
func DecodeKey(b []byte) ([]interface{}, error) {
// EncodeKey encodes args to a slice which can be sorted lexicographically later.
// EncodeKey guarantees the encoded slice is in ascending order for comparison.
func EncodeKey(args ...interface{}) ([]byte, error) {
return encode(args, true)
}
// EncodeValue encodes args to a byte slice which can be decoded later.
// It does not guarantee the order for comparison.
func EncodeValue(args ...interface{}) ([]byte, error) {
return encode(args, false)
}
// Decode decodes values from a byte slice generated with EncodeKey or EncodeValue
// before.
func Decode(b []byte) ([]interface{}, error) {
if len(b) < 1 {
return nil, errors.New("invalid encoded key")
}
@ -143,12 +174,20 @@ func DecodeKey(b []byte) ([]interface{}, error) {
b, v, err = DecodeFloat(b)
case bytesFlag:
b, v, err = DecodeBytes(b)
case compactBytesFlag:
b, v, err = DecodeCompactBytes(b)
case stringFlag:
var r []byte
b, r, err = DecodeBytes(b)
if err == nil {
v = string(r)
}
case compactStringFlag:
var r []byte
b, r, err = DecodeCompactBytes(b)
if err == nil {
v = string(r)
}
case decimalFlag:
b, v, err = DecodeDecimal(b)
case durationFlag:
@ -172,145 +211,3 @@ func DecodeKey(b []byte) ([]interface{}, error) {
return values, nil
}
// EncodeValue encodes args to a byte slice which can be decoded by DecodeValue later.
// It does not guarantee the order for comparison.
func EncodeValue(args ...interface{}) ([]byte, error) {
var b []byte
for _, arg := range args {
switch v := arg.(type) {
case bool:
b = append(b, intFlag)
if v {
b = EncodeInt(b, int64(1))
} else {
b = EncodeInt(b, int64(0))
}
case int:
b = append(b, intFlag)
b = EncodeInt(b, int64(v))
case int8:
b = append(b, intFlag)
b = EncodeInt(b, int64(v))
case int16:
b = append(b, intFlag)
b = EncodeInt(b, int64(v))
case int32:
b = append(b, intFlag)
b = EncodeInt(b, int64(v))
case int64:
b = append(b, intFlag)
b = EncodeInt(b, int64(v))
case uint:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v))
case uint8:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v))
case uint16:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v))
case uint32:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v))
case uint64:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v))
case float32:
b = append(b, floatFlag)
b = EncodeFloat(b, float64(v))
case float64:
b = append(b, floatFlag)
b = EncodeFloat(b, float64(v))
case string:
b = append(b, stringFlag)
b = EncodeCompactBytes(b, []byte(v))
case []byte:
b = append(b, bytesFlag)
b = EncodeCompactBytes(b, v)
case mysql.Time:
b = append(b, stringFlag)
b = EncodeCompactBytes(b, []byte(v.String()))
case mysql.Duration:
// duration may have negative value, so we cannot use String to encode directly.
b = append(b, durationFlag)
b = EncodeInt(b, int64(v.Duration))
case mysql.Decimal:
b = append(b, decimalFlag)
b = EncodeDecimal(b, v)
case mysql.Hex:
b = append(b, intFlag)
b = EncodeInt(b, int64(v.ToNumber()))
case mysql.Bit:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v.ToNumber()))
case mysql.Enum:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v.ToNumber()))
case mysql.Set:
b = append(b, uintFlag)
b = EncodeUint(b, uint64(v.ToNumber()))
case nil:
b = append(b, nilFlag)
default:
return nil, errors.Errorf("unsupport encode type %T", arg)
}
}
return b, nil
}
// DecodeValue decodes values from a byte slice generated with EncodeValue before.
func DecodeValue(b []byte) ([]interface{}, error) {
if len(b) < 1 {
return nil, errors.New("invalid encoded value")
}
var (
flag byte
err error
v interface{}
values = make([]interface{}, 0)
)
for len(b) > 0 {
flag = b[0]
b = b[1:]
switch flag {
case intFlag:
b, v, err = DecodeInt(b)
case uintFlag:
b, v, err = DecodeUint(b)
case floatFlag:
b, v, err = DecodeFloat(b)
case bytesFlag:
b, v, err = DecodeCompactBytes(b)
case stringFlag:
var r []byte
b, r, err = DecodeCompactBytes(b)
if err == nil {
v = string(r)
}
case decimalFlag:
b, v, err = DecodeDecimal(b)
case durationFlag:
var r int64
b, r, err = DecodeInt(b)
if err == nil {
// use max fsp, let outer to do round manually.
v = mysql.Duration{Duration: time.Duration(r), Fsp: mysql.MaxFsp}
}
case nilFlag:
v = nil
default:
return nil, errors.Errorf("invalid encoded value flag %v", flag)
}
if err != nil {
return nil, errors.Trace(err)
}
values = append(values, v)
}
return values, nil
}

10
util/codec/codec_test.go
View File

@ -85,13 +85,13 @@ func (s *testCodecSuite) TestCodecKey(c *C) {
for _, t := range table {
b, err := EncodeKey(t.Input...)
c.Assert(err, IsNil)
args, err := DecodeKey(b)
args, err := Decode(b)
c.Assert(err, IsNil)
c.Assert(args, DeepEquals, t.Expect)
b, err = EncodeValue(t.Input...)
c.Assert(err, IsNil)
args, err = DecodeValue(b)
args, err = Decode(b)
c.Assert(err, IsNil)
c.Assert(args, DeepEquals, t.Expect)
}
@ -493,7 +493,7 @@ func (s *testCodecSuite) TestTime(c *C) {
b, err := EncodeKey(m)
c.Assert(err, IsNil)
v, err := DecodeKey(b)
v, err := Decode(b)
c.Assert(err, IsNil)
c.Assert(v, DeepEquals, []interface{}{t})
}
@ -534,7 +534,7 @@ func (s *testCodecSuite) TestDuration(c *C) {
b, err := EncodeKey(m)
c.Assert(err, IsNil)
v, err := DecodeKey(b)
v, err := Decode(b)
c.Assert(err, IsNil)
m.Fsp = mysql.MaxFsp
c.Assert(v, DeepEquals, []interface{}{m})
@ -586,7 +586,7 @@ func (s *testCodecSuite) TestDecimal(c *C) {
c.Assert(err, IsNil)
b, err := EncodeKey(m)
c.Assert(err, IsNil)
v, err := DecodeKey(b)
v, err := Decode(b)
c.Assert(err, IsNil)
c.Assert(v, HasLen, 1)
vv, ok := v[0].(mysql.Decimal)

2
util/codec/number.go
View File

@ -128,6 +128,7 @@ func DecodeUintDesc(b []byte) ([]byte, uint64, error) {
}
// EncodeVarint appends the encoded value to slice b and returns the appended slice.
// Note that the encoded result is not memcomparable.
func EncodeVarint(b []byte, v int64) []byte {
var data [binary.MaxVarintLen64]byte
n := binary.PutVarint(data[:], v)
@ -148,6 +149,7 @@ func DecodeVarint(b []byte) ([]byte, int64, error) {
}
// EncodeUvarint appends the encoded value to slice b and returns the appended slice.
// Note that the encoded result is not memcomparable.
func EncodeUvarint(b []byte, v uint64) []byte {
var data [binary.MaxVarintLen64]byte
n := binary.PutUvarint(data[:], v)