executor: fix float data accuracy (#20440)
This commit is contained in:
@ -347,14 +347,18 @@ func lengthEncodedIntSize(n uint64) int {
|
||||
}
|
||||
|
||||
const (
|
||||
expFormatBig = 1e15
|
||||
expFormatSmall = 1e-15
|
||||
expFormatBig = 1e15
|
||||
expFormatSmall = 1e-15
|
||||
defaultMySQLPrec = 5
|
||||
)
|
||||
|
||||
func appendFormatFloat(in []byte, fVal float64, prec, bitSize int) []byte {
|
||||
absVal := math.Abs(fVal)
|
||||
var out []byte
|
||||
if prec == types.UnspecifiedLength && (absVal >= expFormatBig || (absVal != 0 && absVal < expFormatSmall)) {
|
||||
if bitSize == 32 {
|
||||
prec = defaultMySQLPrec
|
||||
}
|
||||
out = strconv.AppendFloat(in, fVal, 'e', prec, bitSize)
|
||||
valStr := out[len(in):]
|
||||
// remove the '+' from the string for compatibility.
|
||||
@ -363,6 +367,20 @@ func appendFormatFloat(in []byte, fVal float64, prec, bitSize int) []byte {
|
||||
plusPosInOut := len(in) + plusPos
|
||||
out = append(out[:plusPosInOut], out[plusPosInOut+1:]...)
|
||||
}
|
||||
// remove extra '0'
|
||||
ePos := bytes.IndexByte(valStr, 'e')
|
||||
pointPos := bytes.IndexByte(valStr, '.')
|
||||
ePosInOut := len(in) + ePos
|
||||
pointPosInOut := len(in) + pointPos
|
||||
validPos := ePosInOut
|
||||
for i := ePosInOut - 1; i >= pointPosInOut; i-- {
|
||||
if out[i] == '0' || out[i] == '.' {
|
||||
validPos = i
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
out = append(out[:validPos], out[ePosInOut:]...)
|
||||
} else {
|
||||
out = strconv.AppendFloat(in, fVal, 'f', prec, bitSize)
|
||||
}
|
||||
|
||||
@ -341,6 +341,54 @@ func (s *testUtilSuite) TestAppendFormatFloat(c *C) {
|
||||
-1,
|
||||
64,
|
||||
},
|
||||
{
|
||||
-340282346638528860000000000000000000000,
|
||||
"-3.40282e38",
|
||||
-1,
|
||||
32,
|
||||
},
|
||||
{
|
||||
-34028236,
|
||||
"-34028236.00",
|
||||
2,
|
||||
32,
|
||||
},
|
||||
{
|
||||
-17976921.34,
|
||||
"-17976921.34",
|
||||
2,
|
||||
64,
|
||||
},
|
||||
{
|
||||
-3.402823466e+38,
|
||||
"-3.40282e38",
|
||||
-1,
|
||||
32,
|
||||
},
|
||||
{
|
||||
-1.7976931348623157e308,
|
||||
"-1.7976931348623157e308",
|
||||
-1,
|
||||
64,
|
||||
},
|
||||
{
|
||||
10.0e20,
|
||||
"1e21",
|
||||
-1,
|
||||
32,
|
||||
},
|
||||
{
|
||||
1e20,
|
||||
"1e20",
|
||||
-1,
|
||||
32,
|
||||
},
|
||||
{
|
||||
10.0,
|
||||
"10",
|
||||
-1,
|
||||
32,
|
||||
},
|
||||
}
|
||||
for _, t := range tests {
|
||||
c.Assert(string(appendFormatFloat(nil, t.fVal, t.prec, t.bitSize)), Equals, t.out)
|
||||
|
||||
Reference in New Issue
Block a user