executor: fix float data accuracy (#20440)

This commit is contained in:
Win-Man
2020-10-15 14:26:25 +08:00
committed by GitHub
parent 4745e1586e
commit d8155e9b66
2 changed files with 68 additions and 2 deletions

View File

@ -347,14 +347,18 @@ func lengthEncodedIntSize(n uint64) int {
}
const (
expFormatBig = 1e15
expFormatSmall = 1e-15
expFormatBig = 1e15
expFormatSmall = 1e-15
defaultMySQLPrec = 5
)
func appendFormatFloat(in []byte, fVal float64, prec, bitSize int) []byte {
absVal := math.Abs(fVal)
var out []byte
if prec == types.UnspecifiedLength && (absVal >= expFormatBig || (absVal != 0 && absVal < expFormatSmall)) {
if bitSize == 32 {
prec = defaultMySQLPrec
}
out = strconv.AppendFloat(in, fVal, 'e', prec, bitSize)
valStr := out[len(in):]
// remove the '+' from the string for compatibility.
@ -363,6 +367,20 @@ func appendFormatFloat(in []byte, fVal float64, prec, bitSize int) []byte {
plusPosInOut := len(in) + plusPos
out = append(out[:plusPosInOut], out[plusPosInOut+1:]...)
}
// remove extra '0'
ePos := bytes.IndexByte(valStr, 'e')
pointPos := bytes.IndexByte(valStr, '.')
ePosInOut := len(in) + ePos
pointPosInOut := len(in) + pointPos
validPos := ePosInOut
for i := ePosInOut - 1; i >= pointPosInOut; i-- {
if out[i] == '0' || out[i] == '.' {
validPos = i
} else {
break
}
}
out = append(out[:validPos], out[ePosInOut:]...)
} else {
out = strconv.AppendFloat(in, fVal, 'f', prec, bitSize)
}

View File

@ -341,6 +341,54 @@ func (s *testUtilSuite) TestAppendFormatFloat(c *C) {
-1,
64,
},
{
-340282346638528860000000000000000000000,
"-3.40282e38",
-1,
32,
},
{
-34028236,
"-34028236.00",
2,
32,
},
{
-17976921.34,
"-17976921.34",
2,
64,
},
{
-3.402823466e+38,
"-3.40282e38",
-1,
32,
},
{
-1.7976931348623157e308,
"-1.7976931348623157e308",
-1,
64,
},
{
10.0e20,
"1e21",
-1,
32,
},
{
1e20,
"1e20",
-1,
32,
},
{
10.0,
"10",
-1,
32,
},
}
for _, t := range tests {
c.Assert(string(appendFormatFloat(nil, t.fVal, t.prec, t.bitSize)), Equals, t.out)