From 15ef3f802931d8cf7dc9ebae165bb264f3b09998 Mon Sep 17 00:00:00 2001 From: hui lai <1353307710@qq.com> Date: Fri, 26 Jul 2024 10:11:27 +0800 Subject: [PATCH] [fix](csv reader) fix csv parser incorrect if enclosing line_delimiter (#38347) Csv reader parse data incorrect when data enclosing line_delimiter, for example, line_delimiter is \n and enclose is ', data as follows: ``` 'aaaaaaaaaaaa bbbb' ``` it will be parsed as two columns: `'aaaaaaaaaaaa` and `bbbb',` rather than one column ``` 'aaaaaaaaaaaa bbbb' ``` The reason why this happened is csv reader will not reset result when not match enclose in this `output_buf_read`, causing incorrect truncation was made. Co-authored-by: Xin Liao --- .../exec/format/file_reader/new_plain_text_line_reader.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp index 8dce6e589afde2..75350890aee1df 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp @@ -160,6 +160,11 @@ void EncloseCsvLineReaderContext::_on_pre_match_enclose(const uint8_t* start, si if (_idx != _total_len) { len = update_reading_bound(start); } else { + // It needs to set the result to nullptr for matching enclose may not be read + // after reading the output buf. + // Therefore, if the result is not set to nullptr, + // the parser will consider reading a line as there is a line delimiter. + _result = nullptr; break; } } while (true);