// Copyright 2015 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. package codec import ( "encoding/binary" "time" "github.com/juju/errors" "github.com/pingcap/tidb/mysql" "github.com/pingcap/tidb/util/types" ) // First byte in the encoded value which specifies the encoding type. const ( NilFlag byte = 0 bytesFlag byte = 1 compactBytesFlag byte = 2 intFlag byte = 3 uintFlag byte = 4 floatFlag byte = 5 decimalFlag byte = 6 durationFlag byte = 7 varintFlag byte = 8 uvarintFlag byte = 9 maxFlag byte = 250 ) func encode(b []byte, vals []types.Datum, comparable bool) ([]byte, error) { for _, val := range vals { switch val.Kind() { case types.KindInt64: b = encodeSignedInt(b, val.GetInt64(), comparable) case types.KindUint64: b = encodeUnsignedInt(b, val.GetUint64(), comparable) case types.KindFloat32, types.KindFloat64: b = append(b, floatFlag) b = EncodeFloat(b, val.GetFloat64()) case types.KindString, types.KindBytes: b = encodeBytes(b, val.GetBytes(), comparable) case types.KindMysqlTime: b = append(b, uintFlag) b = EncodeUint(b, val.GetMysqlTime().ToPackedUint()) case types.KindMysqlDuration: // duration may have negative value, so we cannot use String to encode directly. b = append(b, durationFlag) b = EncodeInt(b, int64(val.GetMysqlDuration().Duration)) case types.KindMysqlDecimal: b = append(b, decimalFlag) b = EncodeDecimal(b, val.GetMysqlDecimal()) case types.KindMysqlHex: b = encodeSignedInt(b, int64(val.GetMysqlHex().ToNumber()), comparable) case types.KindMysqlBit: b = encodeUnsignedInt(b, uint64(val.GetMysqlBit().ToNumber()), comparable) case types.KindMysqlEnum: b = encodeUnsignedInt(b, uint64(val.GetMysqlEnum().ToNumber()), comparable) case types.KindMysqlSet: b = encodeUnsignedInt(b, uint64(val.GetMysqlSet().ToNumber()), comparable) case types.KindNull: b = append(b, NilFlag) case types.KindMinNotNull: b = append(b, bytesFlag) case types.KindMaxValue: b = append(b, maxFlag) default: return nil, errors.Errorf("unsupport encode type %d", val.Kind()) } } return b, nil } func encodeBytes(b []byte, v []byte, comparable bool) []byte { if comparable { b = append(b, bytesFlag) b = EncodeBytes(b, v) } else { b = append(b, compactBytesFlag) b = EncodeCompactBytes(b, v) } return b } func encodeSignedInt(b []byte, v int64, comaprable bool) []byte { if comaprable { b = append(b, intFlag) b = EncodeInt(b, v) } else { b = append(b, varintFlag) b = EncodeVarint(b, v) } return b } func encodeUnsignedInt(b []byte, v uint64, comparable bool) []byte { if comparable { b = append(b, uintFlag) b = EncodeUint(b, v) } else { b = append(b, uvarintFlag) b = EncodeUvarint(b, v) } return b } // EncodeKey appends the encoded values to byte slice b, returns the appended // slice. It guarantees the encoded value is in ascending order for comparison. func EncodeKey(b []byte, v ...types.Datum) ([]byte, error) { return encode(b, v, true) } // EncodeValue appends the encoded values to byte slice b, returning the appended // slice. It does not guarantee the order for comparison. func EncodeValue(b []byte, v ...types.Datum) ([]byte, error) { return encode(b, v, false) } // Decode decodes values from a byte slice generated with EncodeKey or EncodeValue // before. func Decode(b []byte) ([]types.Datum, error) { if len(b) < 1 { return nil, errors.New("invalid encoded key") } var ( err error values = make([]types.Datum, 0, 1) ) for len(b) > 0 { var d types.Datum b, d, err = DecodeOne(b) if err != nil { return nil, errors.Trace(err) } values = append(values, d) } return values, nil } // DecodeOne decodes on datum from a byte slice generated with EncodeKey or EncodeValue. func DecodeOne(b []byte) (remain []byte, d types.Datum, err error) { if len(b) < 1 { return nil, d, errors.New("invalid encoded key") } flag := b[0] b = b[1:] switch flag { case intFlag: var v int64 b, v, err = DecodeInt(b) d.SetInt64(v) case uintFlag: var v uint64 b, v, err = DecodeUint(b) d.SetUint64(v) case varintFlag: var v int64 b, v, err = DecodeVarint(b) d.SetInt64(v) case uvarintFlag: var v uint64 b, v, err = DecodeUvarint(b) d.SetUint64(v) case floatFlag: var v float64 b, v, err = DecodeFloat(b) d.SetFloat64(v) case bytesFlag: var v []byte b, v, err = DecodeBytes(b) d.SetBytes(v) case compactBytesFlag: var v []byte b, v, err = DecodeCompactBytes(b) d.SetBytes(v) case decimalFlag: var v mysql.Decimal b, v, err = DecodeDecimal(b) d.SetValue(v) case durationFlag: var r int64 b, r, err = DecodeInt(b) if err == nil { // use max fsp, let outer to do round manually. v := mysql.Duration{Duration: time.Duration(r), Fsp: mysql.MaxFsp} d.SetValue(v) } case NilFlag: default: return b, d, errors.Errorf("invalid encoded key flag %v", flag) } if err != nil { return b, d, errors.Trace(err) } return b, d, nil } // CutOne cuts the first encoded value from b. // It will return the first encoded item and the remains as byte slice. func CutOne(b []byte) (data []byte, remain []byte, err error) { l, err := peek(b) if err != nil { return nil, nil, errors.Trace(err) } return b[:l], b[l:], nil } // peeks the first encoded value from b and returns its length. func peek(b []byte) (length int, err error) { if len(b) < 1 { return 0, errors.New("invalid encoded key") } flag := b[0] length++ b = b[1:] var l int switch flag { case NilFlag: case intFlag, uintFlag, floatFlag, durationFlag: // Those types are stored in 8 bytes. l = 8 case bytesFlag: l, err = peekBytes(b, false) case compactBytesFlag: l, err = peekCompactBytes(b) case decimalFlag: l, err = peekDecimal(b) case varintFlag: l, err = peekVarint(b) case uvarintFlag: l, err = peekUvarint(b) default: return 0, errors.Errorf("invalid encoded key flag %v", flag) } if err != nil { return 0, errors.Trace(err) } length += l return } func peekBytes(b []byte, reverse bool) (int, error) { offset := 0 for { if len(b) < offset+encGroupSize+1 { return 0, errors.New("insufficient bytes to decode value") } // The byte slice is encoded into many groups. // For each group, there are 8 bytes for data and 1 byte for marker. marker := b[offset+encGroupSize] var padCount byte if reverse { padCount = marker } else { padCount = encMarker - marker } offset += encGroupSize + 1 // When padCount is not zero, it means we get the end of the byte slice. if padCount != 0 { break } } return offset, nil } func peekCompactBytes(b []byte) (int, error) { // Get length. v, n := binary.Varint(b) vi := int(v) if n < 0 { return 0, errors.New("value larger than 64 bits") } else if n == 0 { return 0, errors.New("insufficient bytes to decode value") } if len(b) < vi+n { return 0, errors.Errorf("insufficient bytes to decode value, expected length: %v", n) } return n + vi, nil } func peekDecimal(b []byte) (int, error) { if len(b) < 1 || (len(b) < 9 && int64(b[0]) != zeroSign) { return 0, errors.New("cutDecimal error: Invalid decimal data") } if int64(b[0]) == zeroSign { return 1, nil } // The first byte is value sign. // The following 8 bytes are exp value. l := 9 desc := int64(b[l]) == negativeSign // Abs value bytes bl, err := peekBytes(b[l:], desc) if err != nil { return 0, errors.Trace(err) } return l + bl, nil } func peekVarint(b []byte) (int, error) { _, n := binary.Varint(b) if n < 0 { return 0, errors.New("value larger than 64 bits") } return n, nil } func peekUvarint(b []byte) (int, error) { _, n := binary.Uvarint(b) if n < 0 { return 0, errors.New("value larger than 64 bits") } return n, nil }