database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](stream load) (step one)Add arrow data type for stream load (#26709)

Browse Source 
By using the Arrow data format, we can reduce the streamload of data transferred and improve the data import performance
This commit is contained in:
wuwenchi
2023-12-06 23:29:46 +08:00
committed by GitHub
parent 2ca66ff61c
commit 54d062ddee
15 changed files with 452 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/util/load_util.cpp
View File

@ -69,6 +69,8 @@ void LoadUtil::parse_format(const std::string& format_str, const std::string& co
*format_type = TFileFormatType::FORMAT_ORC;
} else if (iequal(format_str, "WAL")) {
*format_type = TFileFormatType::FORMAT_WAL;
} else if (iequal(format_str, "ARROW")) {
*format_type = TFileFormatType::FORMAT_ARROW;
}
return;
}
@ -85,6 +87,7 @@ bool LoadUtil::is_format_support_streaming(TFileFormatType::type format) {
case TFileFormatType::FORMAT_CSV_LZOP:
case TFileFormatType::FORMAT_JSON:
case TFileFormatType::FORMAT_WAL:
case TFileFormatType::FORMAT_ARROW:
return true;
default:
return false;