database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement] Refactor to improve the usability of MemTracker (step2) (#10823)

Browse Source
This commit is contained in:
Xinyi Zou
2022-07-21 17:11:28 +08:00
committed by GitHub
parent 5f6f35e886
commit 4960043f5e
316 changed files with 2145 additions and 4369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
be/src/vec/exec/join/vhash_join_node.cpp
View File

@ -19,7 +19,7 @@
#include "gen_cpp/PlanNodes_types.h"
#include "gutil/strings/substitute.h"
#include "runtime/mem_tracker.h"
#include "runtime/memory/mem_tracker.h"
#include "runtime/runtime_filter_mgr.h"
#include "util/defer_op.h"
#include "vec/core/materialize_block.h"
@ -60,7 +60,6 @@ struct ProcessHashTableBuild {
Defer defer {[&]() {
int64_t bucket_size = hash_table_ctx.hash_table.get_buffer_size_in_cells();
int64_t bucket_bytes = hash_table_ctx.hash_table.get_buffer_size_in_bytes();
_join_node->_hash_table_mem_tracker->consume(bucket_bytes - old_bucket_bytes);
_join_node->_mem_used += bucket_bytes - old_bucket_bytes;
COUNTER_SET(_join_node->_build_buckets_counter, bucket_size);
}};
@ -782,8 +781,7 @@ Status HashJoinNode::init(const TPlanNode& tnode, RuntimeState* state) {
Status HashJoinNode::prepare(RuntimeState* state) {
RETURN_IF_ERROR(ExecNode::prepare(state));
SCOPED_SWITCH_TASK_THREAD_LOCAL_MEM_TRACKER(mem_tracker());
_hash_table_mem_tracker = MemTracker::create_virtual_tracker(-1, "VSetOperationNode:HashTable");
SCOPED_CONSUME_MEM_TRACKER(mem_tracker());
// Build phase
auto build_phase_profile = runtime_profile()->create_child("BuildPhase", true, true);
@ -809,16 +807,13 @@ Status HashJoinNode::prepare(RuntimeState* state) {
_push_compute_timer = ADD_TIMER(runtime_profile(), "PushDownComputeTime");
_build_buckets_counter = ADD_COUNTER(runtime_profile(), "BuildBuckets", TUnit::UNIT);
RETURN_IF_ERROR(
VExpr::prepare(_build_expr_ctxs, state, child(1)->row_desc(), expr_mem_tracker()));
RETURN_IF_ERROR(
VExpr::prepare(_probe_expr_ctxs, state, child(0)->row_desc(), expr_mem_tracker()));
RETURN_IF_ERROR(VExpr::prepare(_build_expr_ctxs, state, child(1)->row_desc()));
RETURN_IF_ERROR(VExpr::prepare(_probe_expr_ctxs, state, child(0)->row_desc()));
// _vother_join_conjuncts are evaluated in the context of the rows produced by this node
if (_vother_join_conjunct_ptr) {
RETURN_IF_ERROR(
(*_vother_join_conjunct_ptr)
->prepare(state, _row_desc_for_other_join_conjunt, expr_mem_tracker()));
(*_vother_join_conjunct_ptr)->prepare(state, _row_desc_for_other_join_conjunt));
}
// right table data types
_right_table_data_types = VectorizedUtils::get_data_types(child(1)->row_desc());
@ -844,8 +839,6 @@ Status HashJoinNode::close(RuntimeState* state) {
(*_vother_join_conjunct_ptr)->close(state);
}
_hash_table_mem_tracker->release(_mem_used);
return ExecNode::close(state);
}
@ -989,8 +982,8 @@ Status HashJoinNode::get_next(RuntimeState* state, Block* output_block, bool* eo
Status HashJoinNode::open(RuntimeState* state) {
START_AND_SCOPE_SPAN(state->get_tracer(), span, "HashJoinNode::open");
SCOPED_TIMER(_runtime_profile->total_time_counter());
SCOPED_SWITCH_TASK_THREAD_LOCAL_MEM_TRACKER(mem_tracker());
RETURN_IF_ERROR(ExecNode::open(state));
SCOPED_CONSUME_MEM_TRACKER(mem_tracker());
RETURN_IF_CANCELLED(state);
RETURN_IF_ERROR(VExpr::open(_build_expr_ctxs, state));
@ -1019,13 +1012,13 @@ Status HashJoinNode::open(RuntimeState* state) {
void HashJoinNode::_hash_table_build_thread(RuntimeState* state, std::promise<Status>* status) {
START_AND_SCOPE_SPAN(state->get_tracer(), span, "HashJoinNode::_hash_table_build_thread");
SCOPED_ATTACH_TASK_THREAD(state, mem_tracker());
SCOPED_ATTACH_TASK(state);
status->set_value(_hash_table_build(state));
}
Status HashJoinNode::_hash_table_build(RuntimeState* state) {
RETURN_IF_ERROR(child(1)->open(state));
SCOPED_SWITCH_THREAD_LOCAL_MEM_TRACKER_ERR_CB("Hash join, while constructing the hash table.");
SCOPED_UPDATE_MEM_EXCEED_CALL_BACK("Hash join, while constructing the hash table.");
SCOPED_TIMER(_build_timer);
MutableBlock mutable_block(child(1)->row_desc().tuple_descriptors());
@ -1043,7 +1036,6 @@ Status HashJoinNode::_hash_table_build(RuntimeState* state) {
RETURN_IF_ERROR_AND_CHECK_SPAN(child(1)->get_next(state, &block, &eos),
child(1)->get_next_span(), eos);
_hash_table_mem_tracker->consume(block.allocated_bytes());
_mem_used += block.allocated_bytes();
if (block.rows() != 0) {