database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bug](scanner) Improve limit query performance on olapScannode and avoid infinite loop (#11301)

Browse Source 
1. Fix a bug that query large column table may cause infinite loop
2. Optimize the query logic with limit, for the case where the limit value is relatively small, reduce the parallelism of the scanner, reduce unnecessary resource consumption, and increase the number of similar queries that the system can carry at the same time, and increase the query speed by more than 60%
This commit is contained in:
Zhengguo Yang
2022-08-01 13:50:12 +08:00
committed by GitHub
parent b35daf0a04
commit 4f5e1601df
8 changed files with 104 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
be/src/vec/exec/volap_scan_node.cpp
View File

@ -1876,35 +1876,58 @@ int VOlapScanNode::_start_scanner_thread_task(RuntimeState* state, int block_per
}
// post volap scanners to thread-pool
PriorityThreadPool* thread_pool = state->exec_env()->scan_thread_pool();
auto cur_span = opentelemetry::trace::Tracer::GetCurrentSpan();
PriorityThreadPool* remote_thread_pool = state->exec_env()->remote_scan_thread_pool();
ThreadPoolToken* thread_token = nullptr;
if (_limit > -1 && _limit < 1024) {
thread_token = state->get_query_fragments_ctx()->get_serial_token();
} else {
thread_token = state->get_query_fragments_ctx()->get_token();
}
auto iter = olap_scanners.begin();
while (iter != olap_scanners.end()) {
PriorityThreadPool::Task task;
task.work_function = [this, scanner = *iter, parent_span = cur_span] {
opentelemetry::trace::Scope scope {parent_span};
this->scanner_thread(scanner);
};
task.priority = _nice;
task.queue_id = state->exec_env()->store_path_to_index((*iter)->scan_disk());
(*iter)->start_wait_worker_timer();
TabletStorageType type = (*iter)->get_storage_type();
bool ret = false;
COUNTER_UPDATE(_scanner_sched_counter, 1);
if (type == TabletStorageType::STORAGE_TYPE_LOCAL) {
ret = thread_pool->offer(task);
} else {
ret = remote_thread_pool->offer(task);
if (thread_token != nullptr) {
while (iter != olap_scanners.end()) {
auto s = thread_token->submit_func([this, scanner = *iter, parent_span = cur_span] {
opentelemetry::trace::Scope scope {parent_span};
this->scanner_thread(scanner);
});
if (s.ok()) {
(*iter)->start_wait_worker_timer();
COUNTER_UPDATE(_scanner_sched_counter, 1);
olap_scanners.erase(iter++);
} else {
LOG(FATAL) << "Failed to assign scanner task to thread pool! " << s.get_error_msg();
}
++_total_assign_num;
}
} else {
PriorityThreadPool* thread_pool = state->exec_env()->scan_thread_pool();
PriorityThreadPool* remote_thread_pool = state->exec_env()->remote_scan_thread_pool();
while (iter != olap_scanners.end()) {
PriorityThreadPool::Task task;
task.work_function = [this, scanner = *iter, parent_span = cur_span] {
opentelemetry::trace::Scope scope {parent_span};
this->scanner_thread(scanner);
};
task.priority = _nice;
task.queue_id = state->exec_env()->store_path_to_index((*iter)->scan_disk());
(*iter)->start_wait_worker_timer();
if (ret) {
olap_scanners.erase(iter++);
} else {
LOG(FATAL) << "Failed to assign scanner task to thread pool!";
TabletStorageType type = (*iter)->get_storage_type();
bool ret = false;
COUNTER_UPDATE(_scanner_sched_counter, 1);
if (type == TabletStorageType::STORAGE_TYPE_LOCAL) {
ret = thread_pool->offer(task);
} else {
ret = remote_thread_pool->offer(task);
}
if (ret) {
olap_scanners.erase(iter++);
} else {
LOG(FATAL) << "Failed to assign scanner task to thread pool!";
}
++_total_assign_num;
}
++_total_assign_num;
}
return assigned_thread_num;