database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improve](multi-table-load) avoid plan and execute too many plan at once (#29951)

Browse Source
This commit is contained in:
HHoflittlefish777
2024-01-16 20:44:54 +08:00
committed by yiguolei
parent 8090a395bd
commit 2fa511f80e
6 changed files with 36 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
be/src/io/fs/multi_table_pipe.cpp
View File

@ -118,10 +118,12 @@ Status MultiTablePipe::dispatch(const std::string& table, const char* data, size
"append failed in unplanned kafka pipe");
++_unplanned_row_cnt;
size_t threshold = config::multi_table_batch_plan_threshold;
if (_unplanned_row_cnt >= threshold) {
LOG(INFO) << fmt::format("unplanned row cnt={} reach threshold={}, plan them",
_unplanned_row_cnt, threshold);
if (_unplanned_row_cnt >= _row_threshold ||
_unplanned_pipes.size() >= _wait_tables_threshold) {
LOG(INFO) << fmt::format(
"unplanned row cnt={} reach row_threshold={} or wait_plan_table_threshold={}, "
"plan them",
_unplanned_row_cnt, _row_threshold, _wait_tables_threshold);
Status st = request_and_exec_plans();
_unplanned_row_cnt = 0;
if (!st.ok()) {