database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

types: fix doDivMod algorithm (#4995)

Browse Source
This commit is contained in:
xiaojian cai
2017-11-03 21:43:18 +08:00
committed by Ewan Chou
parent 2b9a096a94
commit 68fcccdb2c
2 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
util/types/mydecimal.go
View File

@ -1966,7 +1966,7 @@ func doDivMod(from1, from2, to, mod *MyDecimal, fracIncr int) error {
start1 := 0
stop1 := len1
start2 := idx2
stop2 := digitsToWords(idx2+prec2) - 1
stop2 := idx2 + digitsToWords(prec2) - 1
/* removing end zeroes */
for from2.wordBuf[stop2] == 0 && stop2 >= start2 {

4
util/types/mydecimal_test.go
View File

@ -653,6 +653,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) {
{"0.0123456789012345678912345", "9999999999", "0.000000000001234567890246913578148141", nil},
{"10.333000000", "12.34500", "0.837019036046982584042122316", nil},
{"10.000000000060", "2", "5.000000000030000000", nil},
{"51", "0.003430", "14868.804664723032069970", nil},
}
for _, tt := range tests {
var a, b, to MyDecimal
@ -673,6 +674,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) {
{"-234.567", "10.555", "-2.357", nil},
{"234.567", "-10.555", "2.357", nil},
{"99999999999999999999999999999999999999", "3", "0", nil},
{"51", "0.003430", "0.002760", nil},
}
for _, tt := range tests {
var a, b, to MyDecimal
@ -692,6 +694,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) {
{"1.00", "1", "1.000000", nil},
{"1", "1.000", "1.0000", nil},
{"2", "3", "0.6667", nil},
{"51", "0.003430", "14868.8047", nil},
}
for _, tt := range tests {
var a, b, to MyDecimal
@ -709,6 +712,7 @@ func (s *testMyDecimalSuite) TestDivMod(c *C) {
{"1", "2.0", "1.0", nil},
{"1.0", "2", "1.0", nil},
{"2.23", "3", "2.23", nil},
{"51", "0.003430", "0.002760", nil},
}
for _, tt := range tests {
var a, b, to MyDecimal