database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement] Refactor to improve the usability of MemTracker (step2) (#10823)

Browse Source
This commit is contained in:
Xinyi Zou
2022-07-21 17:11:28 +08:00
committed by GitHub
parent 5f6f35e886
commit 4960043f5e
316 changed files with 2145 additions and 4369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
be/src/runtime/runtime_state.cpp
View File

@ -36,8 +36,8 @@
#include "runtime/exec_env.h"
#include "runtime/initial_reservations.h"
#include "runtime/load_path_mgr.h"
#include "runtime/mem_tracker.h"
#include "runtime/mem_tracker_task_pool.h"
#include "runtime/memory/mem_tracker.h"
#include "runtime/memory/mem_tracker_task_pool.h"
#include "runtime/runtime_filter_mgr.h"
#include "util/file_utils.h"
#include "util/load_error_hub.h"
@ -169,6 +169,10 @@ RuntimeState::~RuntimeState() {
if (_buffer_reservation != nullptr) {
_buffer_reservation->Close();
}
// Manually release the child mem tracker before _instance_mem_tracker is destructed.
_obj_pool->clear();
_runtime_filter_mgr.reset();
}
Status RuntimeState::init(const TUniqueId& fragment_instance_id, const TQueryOptions& query_options,
@ -217,13 +221,13 @@ Status RuntimeState::init(const TUniqueId& fragment_instance_id, const TQueryOpt
Status RuntimeState::init_mem_trackers(const TUniqueId& query_id) {
bool has_query_mem_tracker = _query_options.__isset.mem_limit && (_query_options.mem_limit > 0);
int64_t bytes_limit = has_query_mem_tracker ? _query_options.mem_limit : -1;
if (bytes_limit > MemTracker::get_process_tracker()->limit()) {
if (bytes_limit > ExecEnv::GetInstance()->process_mem_tracker()->limit()) {
VLOG_NOTICE << "Query memory limit " << PrettyPrinter::print(bytes_limit, TUnit::BYTES)
<< " exceeds process memory limit of "
<< PrettyPrinter::print(MemTracker::get_process_tracker()->limit(),
<< PrettyPrinter::print(ExecEnv::GetInstance()->process_mem_tracker()->limit(),
TUnit::BYTES)
<< ". Using process memory limit instead";
bytes_limit = MemTracker::get_process_tracker()->limit();
bytes_limit = ExecEnv::GetInstance()->process_mem_tracker()->limit();
}
auto mem_tracker_counter = ADD_COUNTER(&_profile, "MemoryLimit", TUnit::BYTES);
mem_tracker_counter->set(bytes_limit);
@ -239,30 +243,29 @@ Status RuntimeState::init_mem_trackers(const TUniqueId& query_id) {
DCHECK(false);
}
_instance_mem_tracker = MemTracker::create_tracker(
_instance_mem_tracker = std::make_unique<MemTrackerLimiter>(
bytes_limit, "RuntimeState:instance:" + print_id(_fragment_instance_id),
_query_mem_tracker, MemTrackerLevel::INSTANCE, &_profile);
_query_mem_tracker, &_profile);
RETURN_IF_ERROR(init_buffer_poolstate());
_initial_reservations = _obj_pool->add(
new InitialReservations(_obj_pool.get(), _buffer_reservation, nullptr,
_query_options.initial_reservation_total_claims));
_initial_reservations = _obj_pool->add(new InitialReservations(
_obj_pool.get(), _buffer_reservation, _query_options.initial_reservation_total_claims));
RETURN_IF_ERROR(_initial_reservations->Init(_query_id, min_reservation()));
DCHECK_EQ(0, _initial_reservation_refcnt.load());
if (_instance_buffer_reservation != nullptr) {
_instance_buffer_reservation->InitChildTracker(&_profile, _buffer_reservation, nullptr,
_instance_buffer_reservation->InitChildTracker(&_profile, _buffer_reservation,
std::numeric_limits<int64_t>::max());
}
// filter manager depends _instance_mem_tracker
_runtime_filter_mgr->init();
_runtime_filter_mgr->init(_instance_mem_tracker.get());
return Status::OK();
}
Status RuntimeState::init_instance_mem_tracker() {
_instance_mem_tracker = MemTracker::create_tracker(-1, "RuntimeState");
_instance_mem_tracker = std::make_unique<MemTrackerLimiter>(-1, "RuntimeState:instance");
return Status::OK();
}
@ -284,22 +287,16 @@ Status RuntimeState::init_buffer_poolstate() {
VLOG_QUERY << "Buffer pool limit for " << print_id(_query_id) << ": " << max_reservation;
_buffer_reservation = _obj_pool->add(new ReservationTracker);
_buffer_reservation->InitChildTracker(nullptr, exec_env->buffer_reservation(), nullptr,
max_reservation);
_buffer_reservation->InitChildTracker(nullptr, exec_env->buffer_reservation(), max_reservation);
return Status::OK();
}
Status RuntimeState::create_block_mgr() {
DCHECK(_block_mgr2.get() == nullptr);
int64_t block_mgr_limit = _query_mem_tracker->limit();
if (block_mgr_limit < 0) {
block_mgr_limit = std::numeric_limits<int64_t>::max();
}
RETURN_IF_ERROR(BufferedBlockMgr2::create(
this, runtime_profile(), _exec_env->tmp_file_mgr(), block_mgr_limit,
_exec_env->disk_io_mgr()->max_read_buffer_size(), &_block_mgr2));
RETURN_IF_ERROR(BufferedBlockMgr2::create(this, runtime_profile(), _exec_env->tmp_file_mgr(),
_exec_env->disk_io_mgr()->max_read_buffer_size(),
&_block_mgr2));
return Status::OK();
}