[Feature](Memory) Hook TCMalloc new/delete automatically counts to MemTracker (#8476)

Early Design Documentation: https://shimo.im/docs/DT6JXDRkdTvdyV3G

Implement a new way of memory statistics based on TCMalloc New/Delete Hook,
MemTracker and TLS, and it is expected that all memory new/delete/malloc/free
of the BE process can be counted.
This commit is contained in:
Xinyi Zou
2022-03-20 23:06:54 +08:00
committed by GitHub
parent 276792daeb
commit eeae516e37
211 changed files with 1283 additions and 951 deletions

View File

@ -30,6 +30,7 @@
#include "runtime/row_batch.h"
#include "runtime/runtime_filter_mgr.h"
#include "runtime/runtime_state.h"
#include "runtime/thread_context.h"
#include "util/defer_op.h"
#include "util/runtime_profile.h"
@ -147,8 +148,7 @@ Status HashJoinNode::prepare(RuntimeState* state) {
stores_nulls, _is_null_safe_eq_join, id(), mem_tracker(),
state->batch_size() * 2));
_probe_batch.reset(
new RowBatch(child(0)->row_desc(), state->batch_size(), mem_tracker().get()));
_probe_batch.reset(new RowBatch(child(0)->row_desc(), state->batch_size()));
return Status::OK();
}
@ -177,6 +177,7 @@ Status HashJoinNode::close(RuntimeState* state) {
}
void HashJoinNode::build_side_thread(RuntimeState* state, std::promise<Status>* status) {
SCOPED_ATTACH_TASK_THREAD(state, mem_tracker());
status->set_value(construct_hash_table(state));
}
@ -185,7 +186,7 @@ Status HashJoinNode::construct_hash_table(RuntimeState* state) {
// The hash join node needs to keep in memory all build tuples, including the tuple
// row ptrs. The row ptrs are copied into the hash table's internal structure so they
// don't need to be stored in the _build_pool.
RowBatch build_batch(child(1)->row_desc(), state->batch_size(), mem_tracker().get());
RowBatch build_batch(child(1)->row_desc(), state->batch_size());
RETURN_IF_ERROR(child(1)->open(state));
SCOPED_TIMER(_build_timer);