database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement] Refactor to improve the usability of MemTracker (step2) (#10823)

Browse Source
This commit is contained in:
Xinyi Zou
2022-07-21 17:11:28 +08:00
committed by GitHub
parent 5f6f35e886
commit 4960043f5e
316 changed files with 2145 additions and 4369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
be/src/exec/olap_scan_node.cpp
View File

@ -179,7 +179,7 @@ void OlapScanNode::_init_counter(RuntimeState* state) {
Status OlapScanNode::prepare(RuntimeState* state) {
init_scan_profile();
RETURN_IF_ERROR(ScanNode::prepare(state));
SCOPED_SWITCH_TASK_THREAD_LOCAL_MEM_TRACKER(mem_tracker());
SCOPED_CONSUME_MEM_TRACKER(mem_tracker());
// create scanner profile
// create timer
_tablet_counter = ADD_COUNTER(runtime_profile(), "TabletCount ", TUnit::UNIT);
@ -190,8 +190,7 @@ Status OlapScanNode::prepare(RuntimeState* state) {
_init_counter(state);
_tuple_desc = state->desc_tbl().get_tuple_descriptor(_tuple_id);
_scanner_mem_tracker = MemTracker::create_tracker(state->instance_mem_tracker()->limit(),
"Scanners", mem_tracker());
_scanner_mem_tracker = std::make_unique<MemTracker>("Scanners");
if (_tuple_desc == nullptr) {
// TODO: make sure we print all available diagnostic output to our error log
@ -230,9 +229,9 @@ Status OlapScanNode::prepare(RuntimeState* state) {
Status OlapScanNode::open(RuntimeState* state) {
VLOG_CRITICAL << "OlapScanNode::Open";
SCOPED_TIMER(_runtime_profile->total_time_counter());
SCOPED_SWITCH_TASK_THREAD_LOCAL_MEM_TRACKER(mem_tracker());
RETURN_IF_CANCELLED(state);
RETURN_IF_ERROR(ExecNode::open(state));
SCOPED_CONSUME_MEM_TRACKER(mem_tracker());
_resource_info = ResourceTls::get_resource_tls();
@ -258,7 +257,7 @@ Status OlapScanNode::open(RuntimeState* state) {
_runtime_filter_ctxs[i].runtimefilter = runtime_filter;
for (auto ctx : expr_context) {
ctx->prepare(state, row_desc(), _expr_mem_tracker);
ctx->prepare(state, row_desc());
ctx->open(state);
int index = _conjunct_ctxs.size();
_conjunct_ctxs.push_back(ctx);
@ -273,7 +272,7 @@ Status OlapScanNode::open(RuntimeState* state) {
Status OlapScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) {
SCOPED_TIMER(_runtime_profile->total_time_counter());
SCOPED_SWITCH_TASK_THREAD_LOCAL_EXISTED_MEM_TRACKER(mem_tracker());
SCOPED_CONSUME_MEM_TRACKER(mem_tracker());
// check if Canceled.
if (state->is_cancelled()) {
@ -939,7 +938,7 @@ Status OlapScanNode::start_scan_thread(RuntimeState* state) {
}
OlapScanner* scanner =
new OlapScanner(state, this, _olap_scan_node.is_preaggregation,
_need_agg_finalize, *scan_range, _scanner_mem_tracker);
_need_agg_finalize, *scan_range, _scanner_mem_tracker.get());
// add scanner to pool before doing prepare.
// so that scanner can be automatically deconstructed if prepare failed.
_scanner_pool.add(scanner);
@ -1479,7 +1478,7 @@ Status OlapScanNode::normalize_bloom_filter_predicate(SlotDescriptor* slot) {
void OlapScanNode::transfer_thread(RuntimeState* state) {
// scanner open pushdown to scanThread
SCOPED_ATTACH_TASK_THREAD(state, mem_tracker());
SCOPED_ATTACH_TASK(state);
Status status = Status::OK();
for (auto scanner : _olap_scanners) {
status = Expr::clone_if_not_exists(_conjunct_ctxs, state, scanner->conjunct_ctxs());
@ -1507,7 +1506,6 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
_nice = 18 + std::max(0, 2 - (int)_olap_scanners.size() / 5);
std::list<OlapScanner*> olap_scanners;
int64_t mem_limit = _scanner_mem_tracker->limit();
int64_t mem_consume = _scanner_mem_tracker->consumption();
int max_thread = _max_materialized_row_batches;
if (config::doris_scanner_row_num > state->batch_size()) {
@ -1532,7 +1530,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
size_t thread_slot_num = 0;
mem_consume = _scanner_mem_tracker->consumption();
// check limit for total memory and _scan_row_batches memory
if (mem_consume < (mem_limit * 6) / 10 &&
if (mem_consume < (state->instance_mem_tracker()->limit() * 6) / 10 &&
_scan_row_batches_bytes < _max_scanner_queue_size_bytes / 2) {
thread_slot_num = max_thread - assigned_thread_num;
} else {
@ -1648,8 +1646,7 @@ void OlapScanNode::transfer_thread(RuntimeState* state) {
}
void OlapScanNode::scanner_thread(OlapScanner* scanner) {
SCOPED_ATTACH_TASK_THREAD(_runtime_state, mem_tracker());
ADD_THREAD_LOCAL_MEM_TRACKER(scanner->mem_tracker());
SCOPED_ATTACH_TASK(_runtime_state);
Thread::set_self_name("olap_scanner");
if (UNLIKELY(_transfer_done)) {
_scanner_done = true;
@ -1692,7 +1689,7 @@ void OlapScanNode::scanner_thread(OlapScanner* scanner) {
DCHECK(runtime_filter != nullptr);
bool ready = runtime_filter->is_ready();
if (ready) {
runtime_filter->get_prepared_context(&contexts, row_desc(), _expr_mem_tracker);
runtime_filter->get_prepared_context(&contexts, row_desc());
scanner_filter_apply_marks[i] = true;
}
}