database/tidb
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dumpling: use CRLF as line separator (#32149)

Browse Source 
close pingcap/tidb#32105
This commit is contained in:
docsir
2022-02-28 20:13:45 +08:00
committed by GitHub
parent 0c9b0c0ef2
commit c19e899b65
2 changed files with 22 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
dumpling/export/writer_serial_test.go
View File

@ -127,10 +127,10 @@ func TestWriteInsertInCsv(t *testing.T) {
require.Equal(t, uint64(4), n)
require.NoError(t, err)
expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\n" +
"2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\n" +
"3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\n" +
"4,\"female\",\"sarah@mail.com\",\"020-1235\",\"healthy\"\n"
expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\r\n" +
"2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\r\n" +
"3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\r\n" +
"4,\"female\",\"sarah@mail.com\",\"020-1235\",\"healthy\"\r\n"
require.Equal(t, expected, bf.String())
require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels))
require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels))
@ -146,10 +146,10 @@ func TestWriteInsertInCsv(t *testing.T) {
require.Equal(t, uint64(4), n)
require.NoError(t, err)
expected = "1,'male','bob@mail.com','020-1234',\\N\n" +
"2,'female','sarah@mail.com','020-1253','healthy'\n" +
"3,'male','john@mail.com','020-1256','healthy'\n" +
"4,'female','sarah@mail.com','020-1235','healthy'\n"
expected = "1,'male','bob@mail.com','020-1234',\\N\r\n" +
"2,'female','sarah@mail.com','020-1253','healthy'\r\n" +
"3,'male','john@mail.com','020-1256','healthy'\r\n" +
"4,'female','sarah@mail.com','020-1235','healthy'\r\n"
require.Equal(t, expected, bf.String())
require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels))
require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels))
@ -165,10 +165,10 @@ func TestWriteInsertInCsv(t *testing.T) {
require.Equal(t, uint64(4), n)
require.NoError(t, err)
expected = "1;'male';'bob@mail.com';'020-1234';\\N\n" +
"2;'female';'sarah@mail.com';'020-1253';'healthy'\n" +
"3;'male';'john@mail.com';'020-1256';'healthy'\n" +
"4;'female';'sarah@mail.com';'020-1235';'healthy'\n"
expected = "1;'male';'bob@mail.com';'020-1234';\\N\r\n" +
"2;'female';'sarah@mail.com';'020-1253';'healthy'\r\n" +
"3;'male';'john@mail.com';'020-1256';'healthy'\r\n" +
"4;'female';'sarah@mail.com';'020-1235';'healthy'\r\n"
require.Equal(t, expected, bf.String())
require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels))
require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels))
@ -186,11 +186,11 @@ func TestWriteInsertInCsv(t *testing.T) {
require.Equal(t, uint64(4), n)
require.NoError(t, err)
expected = "maidma&;,?magenderma&;,?maemamailma&;,?maphone_numberma&;,?mastatusma\n" +
"1&;,?mamamalema&;,?mabob@mamail.comma&;,?ma020-1234ma&;,?\\N\n" +
"2&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1253ma&;,?mahealthyma\n" +
"3&;,?mamamalema&;,?majohn@mamail.comma&;,?ma020-1256ma&;,?mahealthyma\n" +
"4&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1235ma&;,?mahealthyma\n"
expected = "maidma&;,?magenderma&;,?maemamailma&;,?maphone_numberma&;,?mastatusma\r\n" +
"1&;,?mamamalema&;,?mabob@mamail.comma&;,?ma020-1234ma&;,?\\N\r\n" +
"2&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1253ma&;,?mahealthyma\r\n" +
"3&;,?mamamalema&;,?majohn@mamail.comma&;,?ma020-1256ma&;,?mahealthyma\r\n" +
"4&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1235ma&;,?mahealthyma\r\n"
require.Equal(t, expected, bf.String())
require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels))
require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels))
@ -223,9 +223,9 @@ func TestWriteInsertInCsvReturnsError(t *testing.T) {
require.Equal(t, uint64(3), n)
require.ErrorIs(t, err, rowErr)
expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\n" +
"2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\n" +
"3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\n"
expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\r\n" +
"2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\r\n" +
"3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\r\n"
require.Equal(t, expected, bf.String())
require.Equal(t, float64(0), ReadGauge(finishedRowsGauge, conf.Labels))
require.Equal(t, float64(0), ReadGauge(finishedSizeGauge, conf.Labels))

2
dumpling/export/writer_util.go
View File

@ -344,6 +344,7 @@ func WriteInsertInCsv(pCtx *tcontext.Context, cfg *Config, meta TableMeta, tblIR
bf.Write(opt.separator)
}
}
bf.WriteByte('\r')
bf.WriteByte('\n')
}
wp.currentFileSize += uint64(bf.Len())
@ -359,6 +360,7 @@ func WriteInsertInCsv(pCtx *tcontext.Context, cfg *Config, meta TableMeta, tblIR
counter++
wp.currentFileSize += uint64(bf.Len()-lastBfSize) + 1 // 1 is for "\n"
bf.WriteByte('\r')
bf.WriteByte('\n')
if bf.Len() >= lengthLimit {
select {