database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature] (memory) Switch TLS mem tracker to separate more detailed memory usage (#8605)

Browse Source 
In pr #8476, all memory usage of a process is recorded in the process mem tracker,
and all memory usage of a query is recorded in the query mem tracker,
and it is still necessary to manually call `transfer to` to track the cached memory size.

We hope to separate out more detailed memory usage based on Hook TCMalloc new/delete + TLS mem tracker.

In this pr, the more detailed mem tracker is switched to TLS, which automatically and accurately
counts more detailed memory usage than before.
This commit is contained in:
Xinyi Zou
2022-03-24 14:29:34 +08:00
committed by GitHub
parent 5f606c9d57
commit aaaaae53b5
22 changed files with 202 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
be/src/vec/exec/join/vhash_join_node.cpp
View File

@ -20,6 +20,7 @@
#include "gen_cpp/PlanNodes_types.h"
#include "runtime/mem_tracker.h"
#include "runtime/runtime_filter_mgr.h"
#include "runtime/thread_context.h"
#include "util/defer_op.h"
#include "vec/core/materialize_block.h"
#include "vec/exprs/vexpr.h"
@ -921,6 +922,7 @@ Status HashJoinNode::open(RuntimeState* state) {
Status HashJoinNode::_hash_table_build(RuntimeState* state) {
RETURN_IF_ERROR(child(1)->open(state));
SCOPED_SWITCH_THREAD_LOCAL_MEM_TRACKER_ERR_CB("Hash join, while constructing the hash table.");
SCOPED_TIMER(_build_timer);
MutableBlock mutable_block(child(1)->row_desc().tuple_descriptors());
@ -936,7 +938,6 @@ Status HashJoinNode::_hash_table_build(RuntimeState* state) {
RETURN_IF_ERROR(child(1)->get_next(state, &block, &eos));
_hash_table_mem_tracker->consume(block.allocated_bytes());
_mem_used += block.allocated_bytes();
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Hash join, while getting next from the child 1.");
if (block.rows() != 0) { mutable_block.merge(block); }
@ -947,7 +948,6 @@ Status HashJoinNode::_hash_table_build(RuntimeState* state) {
// TODO:: Rethink may we should do the proess after we recevie all build blocks ?
// which is better.
RETURN_IF_ERROR(_process_build_block(state, _build_blocks[index], index));
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Hash join, while constructing the hash table.");
mutable_block = MutableBlock();
++index;
@ -957,7 +957,6 @@ Status HashJoinNode::_hash_table_build(RuntimeState* state) {
_build_blocks.emplace_back(mutable_block.to_block());
RETURN_IF_ERROR(_process_build_block(state, _build_blocks[index], index));
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Hash join, while constructing the hash table.");
return std::visit(
[&](auto&& arg) -> Status {

8
be/src/vec/exec/vaggregation_node.cpp
View File

@ -22,6 +22,7 @@
#include "exec/exec_node.h"
#include "runtime/mem_pool.h"
#include "runtime/row_batch.h"
#include "runtime/thread_context.h"
#include "util/defer_op.h"
#include "vec/core/block.h"
#include "vec/data_types/data_type_nullable.h"
@ -332,6 +333,7 @@ Status AggregationNode::prepare(RuntimeState* state) {
Status AggregationNode::open(RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::open(state));
SCOPED_SWITCH_THREAD_LOCAL_MEM_TRACKER_ERR_CB("aggregator, while execute open.");
SCOPED_TIMER(_runtime_profile->total_time_counter());
RETURN_IF_ERROR(VExpr::open(_probe_expr_ctxs, state));
@ -356,7 +358,6 @@ Status AggregationNode::open(RuntimeState* state) {
}
RETURN_IF_ERROR(_executor.execute(&block));
_executor.update_memusage();
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "aggregator, while execute open.");
}
return Status::OK();
@ -366,7 +367,9 @@ Status AggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
return Status::NotSupported("Not Implemented Aggregation Node::get_next scalar");
}
Status AggregationNode::get_next(RuntimeState* state, Block* block, bool* eos) { SCOPED_TIMER(_runtime_profile->total_time_counter());
Status AggregationNode::get_next(RuntimeState* state, Block* block, bool* eos) {
SCOPED_SWITCH_THREAD_LOCAL_MEM_TRACKER_ERR_CB("aggregator, while execute get_next.");
SCOPED_TIMER(_runtime_profile->total_time_counter());
if (_is_streaming_preagg) {
bool child_eos = false;
@ -395,7 +398,6 @@ Status AggregationNode::get_next(RuntimeState* state, Block* block, bool* eos) {
}
_executor.update_memusage();
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "aggregator, while execute get_next.");
return Status::OK();
}

4
be/src/vec/exec/vcross_join_node.cpp
View File

@ -23,6 +23,7 @@
#include "gen_cpp/PlanNodes_types.h"
#include "runtime/row_batch.h"
#include "runtime/runtime_state.h"
#include "runtime/thread_context.h"
#include "util/runtime_profile.h"
namespace doris::vectorized {
@ -53,6 +54,7 @@ Status VCrossJoinNode::close(RuntimeState* state) {
Status VCrossJoinNode::construct_build_side(RuntimeState* state) {
// Do a full scan of child(1) and store all build row batches.
RETURN_IF_ERROR(child(1)->open(state));
SCOPED_SWITCH_THREAD_LOCAL_MEM_TRACKER_ERR_CB("Vec Cross join, while getting next from the child 1");
bool eos = false;
while (true) {
@ -70,8 +72,6 @@ Status VCrossJoinNode::construct_build_side(RuntimeState* state) {
_build_blocks.emplace_back(std::move(block));
_block_mem_tracker->consume(mem_usage);
}
// to prevent use too many memory
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Cross join, while getting next from the child 1.");
if (eos) {
break;

6
be/src/vec/exec/vset_operation_node.cpp
View File

@ -17,6 +17,7 @@
#include "vec/exec/vset_operation_node.h"
#include "runtime/thread_context.h"
#include "util/defer_op.h"
#include "vec/exprs/vexpr.h"
namespace doris {
@ -228,6 +229,8 @@ void VSetOperationNode::hash_table_init() {
//build a hash table from child(0)
Status VSetOperationNode::hash_table_build(RuntimeState* state) {
RETURN_IF_ERROR(child(0)->open(state));
SCOPED_SWITCH_THREAD_LOCAL_MEM_TRACKER_ERR_CB(
"Vec Set Operation Node, while constructing the hash table");
Block block;
MutableBlock mutable_block(child(0)->row_desc().tuple_descriptors());
@ -244,7 +247,6 @@ Status VSetOperationNode::hash_table_build(RuntimeState* state) {
_hash_table_mem_tracker->consume(allocated_bytes);
_mem_used += allocated_bytes;
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Set Operation Node, while getting next from the child 0.");
if (block.rows() != 0) { mutable_block.merge(block); }
// make one block for each 4 gigabytes
@ -254,7 +256,6 @@ Status VSetOperationNode::hash_table_build(RuntimeState* state) {
// TODO:: Rethink may we should do the proess after we recevie all build blocks ?
// which is better.
RETURN_IF_ERROR(process_build_block(_build_blocks[index], index));
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Set Operation Node, while constructing the hash table.");
mutable_block = MutableBlock();
++index;
last_mem_used = _mem_used;
@ -263,7 +264,6 @@ Status VSetOperationNode::hash_table_build(RuntimeState* state) {
_build_blocks.emplace_back(mutable_block.to_block());
RETURN_IF_ERROR(process_build_block(_build_blocks[index], index));
RETURN_IF_INSTANCE_LIMIT_EXCEEDED(state, "Set Operation Node, while constructing the hash table.");
return Status::OK();
}