database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](csv-reader) fix column split error when there is escape character (#34364)

Browse Source
This commit is contained in:
Xin Liao
2024-05-02 09:26:23 +08:00
committed by yiguolei
parent ad35968236
commit 971e10a9db
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp
View File

@ -143,13 +143,12 @@ void EncloseCsvLineReaderContext::_on_normal(const uint8_t* start, size_t& len)
}
void EncloseCsvLineReaderContext::_on_pre_match_enclose(const uint8_t* start, size_t& len) {
bool should_escape = false;
do {
do {
if (start[_idx] == _escape) [[unlikely]] {
should_escape = !should_escape;
} else if (should_escape) [[unlikely]] {
should_escape = false;
_should_escape = !_should_escape;
} else if (_should_escape) [[unlikely]] {
_should_escape = false;
} else if (start[_idx] == _enclose) [[unlikely]] {
_state.forward_to(ReaderState::MATCH_ENCLOSE);
++_idx;

2
be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h
View File

@ -135,6 +135,7 @@ public:
inline void refresh_impl() {
_idx = 0;
_should_escape = false;
_result = nullptr;
_column_sep_positions.clear();
_state.reset();
@ -168,6 +169,7 @@ private:
const size_t _column_sep_len;
size_t _idx = 0;
bool _should_escape = false;
const std::string _column_sep;
std::vector<size_t> _column_sep_positions;