database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add scan thread token (#6443)

Browse Source
This commit is contained in:
Mingyu Chen
2021-08-27 10:56:17 +08:00
committed by GitHub
parent 4cfebc35a7
commit 3f2fdd236f
24 changed files with 349 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
be/src/exec/olap_scan_node.cpp
View File

@ -1338,6 +1338,8 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
}
}
ThreadPoolToken* thread_token = state->get_query_fragments_ctx()->get_token();
/*********************************
* 优先级调度基本策略:
* 1. 通过查询拆分的Range个数来确定初始nice值
@ -1348,7 +1350,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
* nice值越大的，越优先获得的查询资源
* 4. 定期提高队列内残留任务的优先级，避免大查询完全饿死
*********************************/
PriorityThreadPool* thread_pool = state->exec_env()->thread_pool();
PriorityThreadPool* thread_pool = state->exec_env()->scan_thread_pool();
_total_assign_num = 0;
_nice = 18 + std::max(0, 2 - (int)_olap_scanners.size() / 5);
std::list<OlapScanner*> olap_scanners;
@ -1413,17 +1415,30 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
}
auto iter = olap_scanners.begin();
while (iter != olap_scanners.end()) {
PriorityThreadPool::Task task;
task.work_function = std::bind(&OlapScanNode::scanner_thread, this, *iter);
task.priority = _nice;
(*iter)->start_wait_worker_timer();
if (thread_pool->offer(task)) {
olap_scanners.erase(iter++);
} else {
LOG(FATAL) << "Failed to assign scanner task to thread pool!";
if (thread_token != nullptr) {
while (iter != olap_scanners.end()) {
auto s = thread_token->submit_func(std::bind(&OlapScanNode::scanner_thread, this, *iter));
if (s.ok()) {
(*iter)->start_wait_worker_timer();
olap_scanners.erase(iter++);
} else {
LOG(FATAL) << "Failed to assign scanner task to thread pool! " << s.get_error_msg();
}
++_total_assign_num;
}
} else {
while (iter != olap_scanners.end()) {
PriorityThreadPool::Task task;
task.work_function = std::bind(&OlapScanNode::scanner_thread, this, *iter);
task.priority = _nice;
(*iter)->start_wait_worker_timer();
if (thread_pool->offer(task)) {
olap_scanners.erase(iter++);
} else {
LOG(FATAL) << "Failed to assign scanner task to thread pool!";
}
++_total_assign_num;
}
++_total_assign_num;
}
RowBatchInterface* scan_batch = NULL;
@ -1466,7 +1481,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
if (NULL != scan_batch) {
add_one_batch(scan_batch);
}
}
} // end of transfer while
state->resource_pool()->release_thread_token(true);
VLOG_CRITICAL << "TransferThread finish.";