database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](Load) Cancel the load job ASAP when encounter unqualified data (#6319)

Browse Source 
This PR mainly changes:

1. Help to Cancel the load job ASAP when encounter unqualified data.
    Solution is described in #6318 .
    Also replace some std::stringstream with fmt::memory_buffer to avoid performance issues.

2. fix a NPE bug when create user with empty host
3. fix compile warning after rebasing the master(vectorization)
This commit is contained in:
Mingyu Chen
2022-01-18 13:13:55 +08:00
committed by GitHub
parent efb4e189df
commit 5fc0a9f40d
58 changed files with 474 additions and 1516 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
be/src/exec/csv_scan_node.cpp
View File

@ -330,8 +330,7 @@ Status CsvScanNode::close(RuntimeState* state) {
if (state->num_rows_load_success() == 0) {
std::stringstream error_msg;
error_msg << "Read zero normal line file. ";
state->append_error_msg_to_file("", error_msg.str(), true);
LOG(INFO) << error_msg.str();
return Status::InternalError(error_msg.str());
}
@ -341,7 +340,7 @@ Status CsvScanNode::close(RuntimeState* state) {
std::stringstream summary_msg;
summary_msg << "error line: " << _num_rows_load_filtered
<< "; normal line: " << state->num_rows_load_success();
state->append_error_msg_to_file("", summary_msg.str(), true);
LOG(INFO) << summary_msg.str();
}
return Status::OK();
@ -526,13 +525,13 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
error_msg << "actual column number is less than schema column number. "
<< "actual number: " << fields.size() << " ,"
<< "schema number: " << _columns.size() << "; ";
_runtime_state->append_error_msg_to_file(line, error_msg.str());
LOG(INFO) << error_msg.str();
return false;
} else if (_hll_column_num == 0 && fields.size() > _columns.size()) {
error_msg << "actual column number is more than schema column number. "
<< "actual number: " << fields.size() << " ,"
<< "schema number: " << _columns.size() << "; ";
_runtime_state->append_error_msg_to_file(line, error_msg.str());
LOG(INFO) << error_msg.str();
return false;
}
@ -557,7 +556,7 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
fields[i].length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
LOG(INFO) << error_msg.str();;
return false;
}
}
@ -582,7 +581,7 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
_default_values[i].length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
LOG(INFO) << error_msg.str();
return false;
}
}
@ -599,7 +598,7 @@ bool CsvScanNode::split_check_fill(const std::string& line, RuntimeState* state)
bool flag = check_and_write_text_slot(column_name, column_type, column_string.c_str(),
column_string.length(), slot, state, &error_msg);
if (flag == false) {
_runtime_state->append_error_msg_to_file(line, error_msg.str());
LOG(INFO) << error_msg.str();
return false;
}
}