From c19e899b6559ffbb0c5ab9c5c95fbf59289f5b07 Mon Sep 17 00:00:00 2001 From: docsir <73268456+docsir@users.noreply.github.com> Date: Mon, 28 Feb 2022 20:13:45 +0800 Subject: [PATCH] dumpling: use CRLF as line separator (#32149) close pingcap/tidb#32105 --- dumpling/export/writer_serial_test.go | 40 +++++++++++++-------------- dumpling/export/writer_util.go | 2 ++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/dumpling/export/writer_serial_test.go b/dumpling/export/writer_serial_test.go index 95b406d51d..bc392adf6c 100644 --- a/dumpling/export/writer_serial_test.go +++ b/dumpling/export/writer_serial_test.go @@ -127,10 +127,10 @@ func TestWriteInsertInCsv(t *testing.T) { require.Equal(t, uint64(4), n) require.NoError(t, err) - expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\n" + - "2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\n" + - "3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\n" + - "4,\"female\",\"sarah@mail.com\",\"020-1235\",\"healthy\"\n" + expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\r\n" + + "2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\r\n" + + "3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\r\n" + + "4,\"female\",\"sarah@mail.com\",\"020-1235\",\"healthy\"\r\n" require.Equal(t, expected, bf.String()) require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels)) require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels)) @@ -146,10 +146,10 @@ func TestWriteInsertInCsv(t *testing.T) { require.Equal(t, uint64(4), n) require.NoError(t, err) - expected = "1,'male','bob@mail.com','020-1234',\\N\n" + - "2,'female','sarah@mail.com','020-1253','healthy'\n" + - "3,'male','john@mail.com','020-1256','healthy'\n" + - "4,'female','sarah@mail.com','020-1235','healthy'\n" + expected = "1,'male','bob@mail.com','020-1234',\\N\r\n" + + "2,'female','sarah@mail.com','020-1253','healthy'\r\n" + + "3,'male','john@mail.com','020-1256','healthy'\r\n" + + "4,'female','sarah@mail.com','020-1235','healthy'\r\n" require.Equal(t, expected, bf.String()) require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels)) require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels)) @@ -165,10 +165,10 @@ func TestWriteInsertInCsv(t *testing.T) { require.Equal(t, uint64(4), n) require.NoError(t, err) - expected = "1;'male';'bob@mail.com';'020-1234';\\N\n" + - "2;'female';'sarah@mail.com';'020-1253';'healthy'\n" + - "3;'male';'john@mail.com';'020-1256';'healthy'\n" + - "4;'female';'sarah@mail.com';'020-1235';'healthy'\n" + expected = "1;'male';'bob@mail.com';'020-1234';\\N\r\n" + + "2;'female';'sarah@mail.com';'020-1253';'healthy'\r\n" + + "3;'male';'john@mail.com';'020-1256';'healthy'\r\n" + + "4;'female';'sarah@mail.com';'020-1235';'healthy'\r\n" require.Equal(t, expected, bf.String()) require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels)) require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels)) @@ -186,11 +186,11 @@ func TestWriteInsertInCsv(t *testing.T) { require.Equal(t, uint64(4), n) require.NoError(t, err) - expected = "maidma&;,?magenderma&;,?maemamailma&;,?maphone_numberma&;,?mastatusma\n" + - "1&;,?mamamalema&;,?mabob@mamail.comma&;,?ma020-1234ma&;,?\\N\n" + - "2&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1253ma&;,?mahealthyma\n" + - "3&;,?mamamalema&;,?majohn@mamail.comma&;,?ma020-1256ma&;,?mahealthyma\n" + - "4&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1235ma&;,?mahealthyma\n" + expected = "maidma&;,?magenderma&;,?maemamailma&;,?maphone_numberma&;,?mastatusma\r\n" + + "1&;,?mamamalema&;,?mabob@mamail.comma&;,?ma020-1234ma&;,?\\N\r\n" + + "2&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1253ma&;,?mahealthyma\r\n" + + "3&;,?mamamalema&;,?majohn@mamail.comma&;,?ma020-1256ma&;,?mahealthyma\r\n" + + "4&;,?mafemamalema&;,?masarah@mamail.comma&;,?ma020-1235ma&;,?mahealthyma\r\n" require.Equal(t, expected, bf.String()) require.Equal(t, float64(len(data)), ReadGauge(finishedRowsGauge, conf.Labels)) require.Equal(t, float64(len(expected)), ReadGauge(finishedSizeGauge, conf.Labels)) @@ -223,9 +223,9 @@ func TestWriteInsertInCsvReturnsError(t *testing.T) { require.Equal(t, uint64(3), n) require.ErrorIs(t, err, rowErr) - expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\n" + - "2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\n" + - "3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\n" + expected := "1,\"male\",\"bob@mail.com\",\"020-1234\",\\N\r\n" + + "2,\"female\",\"sarah@mail.com\",\"020-1253\",\"healthy\"\r\n" + + "3,\"male\",\"john@mail.com\",\"020-1256\",\"healthy\"\r\n" require.Equal(t, expected, bf.String()) require.Equal(t, float64(0), ReadGauge(finishedRowsGauge, conf.Labels)) require.Equal(t, float64(0), ReadGauge(finishedSizeGauge, conf.Labels)) diff --git a/dumpling/export/writer_util.go b/dumpling/export/writer_util.go index b673243c6a..73eb66b0a1 100755 --- a/dumpling/export/writer_util.go +++ b/dumpling/export/writer_util.go @@ -344,6 +344,7 @@ func WriteInsertInCsv(pCtx *tcontext.Context, cfg *Config, meta TableMeta, tblIR bf.Write(opt.separator) } } + bf.WriteByte('\r') bf.WriteByte('\n') } wp.currentFileSize += uint64(bf.Len()) @@ -359,6 +360,7 @@ func WriteInsertInCsv(pCtx *tcontext.Context, cfg *Config, meta TableMeta, tblIR counter++ wp.currentFileSize += uint64(bf.Len()-lastBfSize) + 1 // 1 is for "\n" + bf.WriteByte('\r') bf.WriteByte('\n') if bf.Len() >= lengthLimit { select {