diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
index 1f719c9dd6..e97d3cecf5 100644
--- a/be/src/exec/olap_scanner.cpp
+++ b/be/src/exec/olap_scanner.cpp
@@ -53,7 +53,7 @@ OlapScanner::OlapScanner(RuntimeState* runtime_state, OlapScanNode* parent, bool
_version(-1),
_mem_tracker(MemTracker::CreateTracker(
runtime_state->fragment_mem_tracker()->limit(), "OlapScanner",
- runtime_state->fragment_mem_tracker(), true, true, MemTrackerLevel::DEBUG)) {
+ runtime_state->fragment_mem_tracker(), true, true, MemTrackerLevel::VERBOSE)) {
_rows_read_counter = parent->rows_read_counter();
_rows_pushed_cond_filtered_counter = parent->_rows_pushed_cond_filtered_counter;
}
diff --git a/be/src/http/action/compaction_action.h b/be/src/http/action/compaction_action.h
index 5a7b936c4b..57513de50c 100644
--- a/be/src/http/action/compaction_action.h
+++ b/be/src/http/action/compaction_action.h
@@ -42,7 +42,7 @@ public:
CompactionAction(CompactionActionType type)
: _type(type) {
_compaction_mem_tracker = type == RUN_COMPACTION ?
- MemTracker::CreateTracker(-1, "ManualCompaction") : nullptr;
+ MemTracker::CreateTracker(-1, "ManualCompaction", nullptr, false, false, MemTrackerLevel::TASK) : nullptr;
}
virtual ~CompactionAction() {}
diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
index d11991d36f..d8416970dc 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -129,7 +129,8 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
" data-search='true' "
" class='table table-striped'>\n";
(*output) << ""
- "| Id | "
+ "Id | "
"Parent | "
"Limit | "
"Current Consumption | "
"Peak Consumption | ";
+ ">Peak Consumption"
+ "Use Count |
";
(*output) << "\n";
std::vector> trackers;
@@ -147,10 +150,11 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
string limit_str = tracker->limit() == -1 ? "none" : ItoaKMGT(tracker->limit());
string current_consumption_str = ItoaKMGT(tracker->consumption());
string peak_consumption_str = ItoaKMGT(tracker->peak_consumption());
+ int64_t use_count = tracker.use_count();
(*output) << strings::Substitute(
"| $0 | $1 | $2 | " // id, parent, limit
- "$3 | $4 |
\n", // current, peak
- tracker->label(), parent, limit_str, current_consumption_str, peak_consumption_str);
+ "$3 | $4 | $5 | \n", // current, peak
+ tracker->label(), parent, limit_str, current_consumption_str, peak_consumption_str, use_count);
}
(*output) << "\n";
}
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 383ae9ced1..f735be8110 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -28,7 +28,7 @@ namespace doris {
Compaction::Compaction(TabletSharedPtr tablet, const std::string& label,
const std::shared_ptr& parent_tracker)
- : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker, true, false)),
+ : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker, true, false, MemTrackerLevel::TASK)),
_readers_tracker(MemTracker::CreateTracker(-1, "CompactionReaderTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker,
true, false)),
_writer_tracker(MemTracker::CreateTracker(-1, "CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker,
@@ -177,7 +177,7 @@ OLAPStatus Compaction::construct_input_rowset_readers() {
RETURN_NOT_OK(rowset->create_reader(
MemTracker::CreateTracker(
-1, "Compaction:RowsetReader:" + rowset->rowset_id().to_string(),
- _readers_tracker, true, true, MemTrackerLevel::DEBUG),
+ _readers_tracker, true, true),
&rs_reader));
_input_rs_readers.push_back(std::move(rs_reader));
}
diff --git a/be/src/olap/fs/file_block_manager.cpp b/be/src/olap/fs/file_block_manager.cpp
index 2f49136a01..f62f95dbaf 100644
--- a/be/src/olap/fs/file_block_manager.cpp
+++ b/be/src/olap/fs/file_block_manager.cpp
@@ -368,8 +368,8 @@ Status FileReadableBlock::readv(uint64_t offset, const Slice* results, size_t re
FileBlockManager::FileBlockManager(Env* env, BlockManagerOptions opts)
: _env(DCHECK_NOTNULL(env)),
_opts(std::move(opts)),
- _mem_tracker(
- MemTracker::CreateTracker(-1, "FileBlockManager", _opts.parent_mem_tracker)) {
+ _mem_tracker(MemTracker::CreateTracker(-1, "FileBlockManager", _opts.parent_mem_tracker,
+ false, false, MemTrackerLevel::OVERVIEW)) {
if (_opts.enable_metric) {
_metrics.reset(new internal::BlockManagerMetrics());
}
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index f420cc5927..b5919091b2 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -412,7 +412,7 @@ uint32_t ShardedLRUCache::_shard(uint32_t hash) {
ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity, std::shared_ptr parent)
: _name(name), _last_id(1),
- _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false)) {
+ _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false, MemTrackerLevel::OVERVIEW)) {
const size_t per_shard = (total_capacity + (kNumShards - 1)) / kNumShards;
for (int s = 0; s < kNumShards; s++) {
_shards[s].set_capacity(per_shard);
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index 6ce9faba6b..056ad63192 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -29,7 +29,7 @@ void StoragePageCache::create_global_cache(size_t capacity, int32_t index_cache_
StoragePageCache::StoragePageCache(size_t capacity, int32_t index_cache_percentage)
: _index_cache_percentage(index_cache_percentage),
- _mem_tracker(MemTracker::CreateTracker(capacity, "StorgePageCache")) {
+ _mem_tracker(MemTracker::CreateTracker(capacity, "StorgePageCache", nullptr, true, true, MemTrackerLevel::OVERVIEW)) {
if (index_cache_percentage == 0) {
_data_page_cache = std::unique_ptr(new_lru_cache("DataPageCache", capacity, _mem_tracker));
} else if (index_cache_percentage == 100) {
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index fc03a7c094..a7a7dcb357 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -50,7 +50,7 @@ OLAPStatus BetaRowset::do_load(bool /*use_cache*/, std::shared_ptr p
for (int seg_id = 0; seg_id < num_segments(); ++seg_id) {
std::string seg_path = segment_file_path(_rowset_path, rowset_id(), seg_id);
std::shared_ptr segment;
- auto s = segment_v2::Segment::open(seg_path, seg_id, _schema, parent, &segment);
+ auto s = segment_v2::Segment::open(seg_path, seg_id, _schema, &segment);
if (!s.ok()) {
LOG(WARNING) << "failed to open segment " << seg_path << " under rowset " << unique_id()
<< " : " << s.to_string();
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index d0e74966f0..ee13dad67c 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -91,7 +91,7 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
std::vector> seg_iterators;
for (auto& seg_ptr : _rowset->_segments) {
std::unique_ptr iter;
- auto s = seg_ptr->new_iterator(schema, read_options, &iter);
+ auto s = seg_ptr->new_iterator(schema, read_options, _parent_tracker, &iter);
if (!s.ok()) {
LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() << "]: " << s.to_string();
return OLAP_ERR_ROWSET_READER_INIT;
diff --git a/be/src/olap/rowset/segment_reader.cpp b/be/src/olap/rowset/segment_reader.cpp
index 38d3d1fa93..8314d0af1d 100644
--- a/be/src/olap/rowset/segment_reader.cpp
+++ b/be/src/olap/rowset/segment_reader.cpp
@@ -58,7 +58,7 @@ SegmentReader::SegmentReader(const std::string file, SegmentGroup* segment_group
_is_using_mmap(false),
_is_data_loaded(false),
_buffer_size(0),
- _tracker(MemTracker::CreateTracker(-1, "SegmentReader:" + file, parent_tracker)),
+ _tracker(MemTracker::CreateTracker(-1, "SegmentReader:" + file, parent_tracker, false)),
_mem_pool(new MemPool(_tracker.get())),
_shared_buffer(NULL),
_lru_cache(lru_cache),
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 992d8c414d..fbfd52fb74 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -27,6 +27,7 @@
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/rowset/segment_v2/segment_iterator.h"
#include "olap/rowset/segment_v2/segment_writer.h" // k_segment_magic_length
+#include "olap/storage_engine.h"
#include "olap/tablet_schema.h"
#include "util/crc32c.h"
#include "util/slice.h" // Slice
@@ -37,16 +38,22 @@ namespace segment_v2 {
using strings::Substitute;
Status Segment::open(std::string filename, uint32_t segment_id, const TabletSchema* tablet_schema,
- std::shared_ptr parent, std::shared_ptr* output) {
- std::shared_ptr segment(new Segment(std::move(filename), segment_id, tablet_schema, std::move(parent)));
+ std::shared_ptr* output) {
+ std::shared_ptr segment(new Segment(std::move(filename), segment_id, tablet_schema));
RETURN_IF_ERROR(segment->_open());
output->swap(segment);
return Status::OK();
}
-Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema, std::shared_ptr parent)
+Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema)
: _fname(std::move(fname)), _segment_id(segment_id),
- _tablet_schema(tablet_schema), _mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent), false, true)) {}
+ _tablet_schema(tablet_schema) {
+#ifndef BE_TEST
+ _mem_tracker = MemTracker::CreateTracker(-1, "Segment", StorageEngine::instance()->tablet_mem_tracker(), false);
+#else
+ _mem_tracker = MemTracker::CreateTracker(-1, "Segment", nullptr, false);
+#endif
+}
Segment::~Segment() {
_mem_tracker->Release(_mem_tracker->consumption());
@@ -59,6 +66,7 @@ Status Segment::_open() {
}
Status Segment::new_iterator(const Schema& schema, const StorageReadOptions& read_options,
+ std::shared_ptr parent,
std::unique_ptr* iter) {
DCHECK_NOTNULL(read_options.stats);
read_options.stats->total_segment_number++;
@@ -80,7 +88,7 @@ Status Segment::new_iterator(const Schema& schema, const StorageReadOptions& rea
}
RETURN_IF_ERROR(_load_index());
- iter->reset(new SegmentIterator(this->shared_from_this(), schema, _mem_tracker));
+ iter->reset(new SegmentIterator(this->shared_from_this(), schema, parent));
iter->get()->init(read_options);
return Status::OK();
}
@@ -188,7 +196,7 @@ Status Segment::_create_column_readers() {
return Status::OK();
}
-Status Segment::new_column_iterator(uint32_t cid, ColumnIterator** iter) {
+Status Segment::new_column_iterator(uint32_t cid, std::shared_ptr parent, ColumnIterator** iter) {
if (_column_readers[cid] == nullptr) {
const TabletColumn& tablet_column = _tablet_schema->column(cid);
if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) {
@@ -200,7 +208,7 @@ Status Segment::new_column_iterator(uint32_t cid, ColumnIterator** iter) {
tablet_column.has_default_value(), tablet_column.default_value(),
tablet_column.is_nullable(), type_info, tablet_column.length()));
ColumnIteratorOptions iter_opts;
- iter_opts.mem_tracker = MemTracker::CreateTracker(-1, "DefaultColumnIterator", _mem_tracker, false);
+ iter_opts.mem_tracker = MemTracker::CreateTracker(-1, "DefaultColumnIterator", parent, false);
RETURN_IF_ERROR(default_value_iter->init(iter_opts));
*iter = default_value_iter.release();
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index cc883b19fe..0cfd40889a 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -60,18 +60,19 @@ using SegmentSharedPtr = std::shared_ptr;
class Segment : public std::enable_shared_from_this {
public:
static Status open(std::string filename, uint32_t segment_id, const TabletSchema* tablet_schema,
- std::shared_ptr parent, std::shared_ptr* output);
+ std::shared_ptr* output);
~Segment();
Status new_iterator(const Schema& schema, const StorageReadOptions& read_options,
+ std::shared_ptr parent,
std::unique_ptr* iter);
uint64_t id() const { return _segment_id; }
uint32_t num_rows() const { return _footer.num_rows(); }
- Status new_column_iterator(uint32_t cid, ColumnIterator** iter);
+ Status new_column_iterator(uint32_t cid, std::shared_ptr parent, ColumnIterator** iter);
Status new_bitmap_index_iterator(uint32_t cid, BitmapIndexIterator** iter);
@@ -104,7 +105,7 @@ public:
private:
DISALLOW_COPY_AND_ASSIGN(Segment);
- Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema, std::shared_ptr parent);
+ Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema);
// open segment file and read the minimum amount of necessary information (footer)
Status _open();
Status _parse_footer();
@@ -119,6 +120,8 @@ private:
uint32_t _segment_id;
const TabletSchema* _tablet_schema;
+ // This mem tracker is only for tracking memory use by segment meta data such as footer or index page.
+ // The memory consumed by querying is tracked in segment iterator.
std::shared_ptr _mem_tracker;
SegmentFooterPB _footer;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 859cca50c4..50e8febb96 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -99,7 +99,7 @@ SegmentIterator::SegmentIterator(std::shared_ptr segment, const Schema&
_lazy_materialization_read(false),
_inited(false) {
// use for count the mem use of ColumnIterator
- _mem_tracker = MemTracker::CreateTracker(-1, "SegmentIterator", std::move(parent), 0);
+ _mem_tracker = MemTracker::CreateTracker(-1, "SegmentIterator", std::move(parent), false);
}
SegmentIterator::~SegmentIterator() {
@@ -194,7 +194,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra
// create used column iterator
for (auto cid : _seek_schema->column_ids()) {
if (_column_iterators[cid] == nullptr) {
- RETURN_IF_ERROR(_segment->new_column_iterator(cid, &_column_iterators[cid]));
+ RETURN_IF_ERROR(_segment->new_column_iterator(cid, _mem_tracker, &_column_iterators[cid]));
ColumnIteratorOptions iter_opts;
iter_opts.stats = _opts.stats;
iter_opts.rblock = _rblock.get();
@@ -322,7 +322,7 @@ Status SegmentIterator::_init_return_column_iterators() {
}
for (auto cid : _schema.column_ids()) {
if (_column_iterators[cid] == nullptr) {
- RETURN_IF_ERROR(_segment->new_column_iterator(cid, &_column_iterators[cid]));
+ RETURN_IF_ERROR(_segment->new_column_iterator(cid, _mem_tracker, &_column_iterators[cid]));
ColumnIteratorOptions iter_opts;
iter_opts.stats = _opts.stats;
iter_opts.use_page_cache = _opts.use_page_cache;
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 7d69f604a9..70b5d15c7d 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -1361,7 +1361,7 @@ bool SchemaChangeWithSorting::_external_sorting(vector& src_row
}
SchemaChangeHandler::SchemaChangeHandler()
- : _mem_tracker(MemTracker::CreateTracker(-1, "SchemaChange")) {
+ : _mem_tracker(MemTracker::CreateTracker(-1, "SchemaChange", StorageEngine::instance()->schema_change_mem_tracker())) {
REGISTER_HOOK_METRIC(schema_change_mem_consumption,
[this]() { return _mem_tracker->consumption(); });
}
@@ -1474,7 +1474,7 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe
reader_context.seek_columns = &return_columns;
auto mem_tracker = MemTracker::CreateTracker(-1, "AlterTablet:" + std::to_string(base_tablet->tablet_id()) + "-"
- + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false, MemTrackerLevel::DEBUG);
+ + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false, MemTrackerLevel::TASK);
do {
// get history data to be converted and it will check if there is hold in base tablet
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index e5eaf71e3f..fd20d31a0a 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -113,7 +113,9 @@ StorageEngine::StorageEngine(const EngineOptions& options)
_index_stream_lru_cache(NULL),
_file_cache(nullptr),
_compaction_mem_tracker(
- MemTracker::CreateTracker(-1, "AutoCompaction")),
+ MemTracker::CreateTracker(-1, "AutoCompaction", nullptr, false, false, MemTrackerLevel::OVERVIEW)),
+ _tablet_mem_tracker(
+ MemTracker::CreateTracker(-1, "TabletHeader", nullptr, false, false, MemTrackerLevel::OVERVIEW)),
_stop_background_threads_latch(1),
_tablet_manager(new TabletManager(config::tablet_map_shard_size)),
_txn_manager(new TxnManager(config::txn_map_shard_size, config::txn_shard_size)),
@@ -1028,14 +1030,14 @@ bool StorageEngine::check_rowset_id_in_unused_rowsets(const RowsetId& rowset_id)
void StorageEngine::create_cumulative_compaction(
TabletSharedPtr best_tablet, std::shared_ptr& cumulative_compaction) {
- std::string tracker_label = "StorageEngine:CumulativeCompaction:" + std::to_string(syscall(__NR_gettid));
+ std::string tracker_label = "StorageEngine:CumulativeCompaction:" + std::to_string(best_tablet->tablet_id());
cumulative_compaction.reset(
new CumulativeCompaction(best_tablet, tracker_label, _compaction_mem_tracker));
}
void StorageEngine::create_base_compaction(TabletSharedPtr best_tablet,
std::shared_ptr& base_compaction) {
- std::string tracker_label = "StorageEngine:BaseCompaction:" + std::to_string(syscall(__NR_gettid));
+ std::string tracker_label = "StorageEngine:BaseCompaction:" + std::to_string(best_tablet->tablet_id());
base_compaction.reset(new BaseCompaction(best_tablet, tracker_label, _compaction_mem_tracker));
}
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 8980e82b02..26afb53922 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -179,6 +179,9 @@ public:
Status get_compaction_status_json(std::string* result);
+ std::shared_ptr tablet_mem_tracker() { return _tablet_mem_tracker; }
+ std::shared_ptr schema_change_mem_tracker() { return _schema_change_mem_tracker; }
+
private:
// Instance should be inited from `static open()`
// MUST NOT be called in other circumstances.
@@ -306,6 +309,7 @@ private:
std::unordered_map _unused_rowsets;
std::shared_ptr _compaction_mem_tracker;
+ std::shared_ptr _tablet_mem_tracker;
std::shared_ptr _schema_change_mem_tracker;
CountDownLatch _stop_background_threads_latch;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index b1e7cf3429..09a9e87db7 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -71,7 +71,7 @@ static bool _cmp_tablet_by_create_time(const TabletSharedPtr& a, const TabletSha
}
TabletManager::TabletManager(int32_t tablet_map_lock_shard_size)
- : _mem_tracker(MemTracker::CreateTracker(-1, "TabletManager", nullptr, false)),
+ : _mem_tracker(MemTracker::CreateTracker(-1, "TabletMeta", nullptr, false, false, MemTrackerLevel::OVERVIEW)),
_tablets_shards_size(tablet_map_lock_shard_size),
_tablets_shards_mask(tablet_map_lock_shard_size - 1),
_last_update_stat_ms(0) {
diff --git a/be/src/runtime/buffered_block_mgr2.cc b/be/src/runtime/buffered_block_mgr2.cc
index 9621092500..48d4a76895 100644
--- a/be/src/runtime/buffered_block_mgr2.cc
+++ b/be/src/runtime/buffered_block_mgr2.cc
@@ -1288,8 +1288,6 @@ void BufferedBlockMgr2::init(DiskIoMgr* io_mgr, RuntimeProfile* parent_profile,
_integrity_check_timer = ADD_TIMER(_profile.get(), "TotalIntegrityCheckTime");
// Create a new mem_tracker and allocate buffers.
- // _mem_tracker.reset(new MemTracker(
- // profile(), mem_limit, -1, "Block Manager", parent_tracker));
_mem_tracker = MemTracker::CreateTracker(mem_limit, "BufferedBlockMgr2", parent_tracker);
_initialized = true;
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 5968a8e1a9..2fcf901124 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -140,6 +140,9 @@ public:
PluginMgr* plugin_mgr() { return _plugin_mgr; }
+ // The root tracker should be set before calling ExecEnv::init();
+ void set_root_mem_tracker(std::shared_ptr root_tracker);
+
private:
Status _init(const std::vector& store_paths);
void _destroy();
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 7bd31a32e4..63095d57c8 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -185,8 +185,8 @@ Status ExecEnv::_init_mem_tracker() {
return Status::InternalError(ss.str());
}
- _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Query", MemTracker::GetRootTracker(),
- false, false);
+ _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Process", MemTracker::GetRootTracker(),
+ false, false, MemTrackerLevel::OVERVIEW);
LOG(INFO) << "Using global memory limit: " << PrettyPrinter::print(bytes_limit, TUnit::BYTES);
RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker));
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index d6ade09e89..43f8a883dd 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -27,7 +27,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t mem_limit, int64_t tim
const std::shared_ptr& mem_tracker)
: _load_id(load_id), _timeout_s(timeout_s) {
_mem_tracker = MemTracker::CreateTracker(
- mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker, true, false);
+ mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker, true, false, MemTrackerLevel::TASK);
// _last_updated_time should be set before being inserted to
// _load_channels in load_channel_mgr, or it may be erased
// immediately by gc thread.
diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp
index 15e2c08fa0..f0110aaa3f 100644
--- a/be/src/runtime/load_channel_mgr.cpp
+++ b/be/src/runtime/load_channel_mgr.cpp
@@ -85,7 +85,7 @@ LoadChannelMgr::~LoadChannelMgr() {
Status LoadChannelMgr::init(int64_t process_mem_limit) {
int64_t load_mem_limit = calc_process_max_load_memory(process_mem_limit);
- _mem_tracker = MemTracker::CreateTracker(load_mem_limit, "Load");
+ _mem_tracker = MemTracker::CreateTracker(load_mem_limit, "LoadChannelMgr", nullptr, true, false, MemTrackerLevel::OVERVIEW);
REGISTER_HOOK_METRIC(load_mem_consumption, [this]() {
return _mem_tracker->consumption();
});
diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp
index 9177e72c46..ea0a16984a 100644
--- a/be/src/runtime/mem_tracker.cpp
+++ b/be/src/runtime/mem_tracker.cpp
@@ -72,7 +72,7 @@ static std::shared_ptr root_tracker;
static GoogleOnceType root_tracker_once = GOOGLE_ONCE_INIT;
void MemTracker::CreateRootTracker() {
- root_tracker.reset(new MemTracker(-1, "Root"));
+ root_tracker.reset(new MemTracker(nullptr, -1, "Root", nullptr, true, MemTrackerLevel::OVERVIEW));
root_tracker->Init();
}
@@ -143,7 +143,7 @@ std::shared_ptr MemTracker::CreateTracker(int64_t byte_limit, const
}
MemTracker::MemTracker(int64_t byte_limit, const std::string& label)
- : MemTracker(nullptr, byte_limit, label, std::shared_ptr(), true, MemTrackerLevel::RELEASE) {}
+ : MemTracker(nullptr, byte_limit, label, std::shared_ptr(), true, MemTrackerLevel::VERBOSE) {}
MemTracker::MemTracker(RuntimeProfile* profile, int64_t byte_limit, const string& label,
const std::shared_ptr& parent, bool log_usage_if_zero, MemTrackerLevel level)
diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h
index 142e611299..b938f6e12d 100644
--- a/be/src/runtime/mem_tracker.h
+++ b/be/src/runtime/mem_tracker.h
@@ -42,7 +42,7 @@ enum class MemLimit { HARD, SOFT };
/// The Level use to decide whether to show it in web page
/// each MemTracker have a Level equals to parent, only be set explicit
-enum class MemTrackerLevel {RELEASE = 0, DEBUG};
+enum class MemTrackerLevel {OVERVIEW = 0, TASK, VERBOSE};
class ObjectPool;
class MemTracker;
@@ -92,11 +92,11 @@ public:
static std::shared_ptr CreateTracker(
int64_t byte_limit = -1, const std::string& label = std::string(),
std::shared_ptr parent = std::shared_ptr(),
- bool log_usage_if_zero = true, bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::RELEASE);
+ bool log_usage_if_zero = true, bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::VERBOSE);
static std::shared_ptr CreateTracker(
RuntimeProfile* profile, int64_t byte_limit, const std::string& label = std::string(),
- const std::shared_ptr& parent = std::shared_ptr(), bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::RELEASE);
+ const std::shared_ptr& parent = std::shared_ptr(), bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::VERBOSE);
// this is used for creating an orphan mem tracker, or for unit test.
// If a mem tracker has parent, it should be created by `CreateTracker()`
diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp
index 3bd8a1be72..2df18a5d1f 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -132,7 +132,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request,
_mem_tracker = MemTracker::CreateTracker(bytes_limit,
"PlanFragmentExecutor:" + print_id(_query_id) + ":" +
print_id(params.fragment_instance_id),
- _exec_env->process_mem_tracker(), true, false);
+ _exec_env->process_mem_tracker(), true, false, MemTrackerLevel::TASK);
_runtime_state->set_fragment_mem_tracker(_mem_tracker);
LOG(INFO) << "Using query memory limit: " << PrettyPrinter::print(bytes_limit, TUnit::BYTES);
diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp b/be/test/olap/rowset/segment_v2/segment_test.cpp
index 2efb213493..cbb3ac8c72 100644
--- a/be/test/olap/rowset/segment_v2/segment_test.cpp
+++ b/be/test/olap/rowset/segment_v2/segment_test.cpp
@@ -136,7 +136,7 @@ protected:
ASSERT_TRUE(st.ok());
ASSERT_TRUE(wblock->close().ok());
- st = Segment::open(filename, 0, &query_schema, MemTracker::CreateTracker(-1, "Segment", nullptr, false), res);
+ st = Segment::open(filename, 0, &query_schema, res);
ASSERT_TRUE(st.ok());
ASSERT_EQ(nrows, (*res)->num_rows());
}
@@ -164,7 +164,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -220,7 +220,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), true);
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -246,7 +246,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false, nullptr, false);
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
ASSERT_TRUE(iter->next_batch(&block).is_end_of_file());
@@ -275,7 +275,7 @@ TEST_F(SegmentReaderWriterTest, normal) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), false);
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
ASSERT_TRUE(iter->next_batch(&block).is_end_of_file());
@@ -312,7 +312,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -336,7 +336,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -356,7 +356,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -388,7 +388,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) {
read_opts.stats = &stats;
std::unique_ptr iter;
- ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok());
+ ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok());
RowBlockV2 block(read_schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -450,7 +450,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
read_opts.conditions = conditions.get();
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1);
@@ -473,7 +473,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
read_opts.conditions = conditions.get();
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -533,7 +533,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
read_opts.delete_conditions.push_back(delete_conditions.get());
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -581,7 +581,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) {
ASSERT_EQ(OLAP_SUCCESS, conditions->append_condition(condition));
read_opts.conditions = conditions.get();
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).is_end_of_file());
@@ -674,7 +674,7 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -724,7 +724,7 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -817,7 +817,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
{
std::shared_ptr segment;
- st = Segment::open(fname, 0, tablet_schema.get(), MemTracker::CreateTracker(-1, "Segment", nullptr, false), &segment);
+ st = Segment::open(fname, 0, tablet_schema.get(), &segment);
ASSERT_TRUE(st.ok());
ASSERT_EQ(4096, segment->num_rows());
Schema schema(*tablet_schema);
@@ -827,7 +827,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
StorageReadOptions read_opts;
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -880,7 +880,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false, nullptr, false);
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
st = iter->next_batch(&block);
@@ -913,7 +913,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.stats = &stats;
read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), false);
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 100);
st = iter->next_batch(&block);
@@ -937,7 +937,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.conditions = conditions.get();
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
int left = 4 * 1024;
@@ -994,7 +994,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
read_opts.conditions = conditions.get();
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
@@ -1033,7 +1033,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -1055,7 +1055,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -1077,7 +1077,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_FALSE(iter->next_batch(&block).ok());
@@ -1101,7 +1101,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);
ASSERT_TRUE(iter->next_batch(&block).ok());
@@ -1124,7 +1124,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) {
read_opts.stats = &stats;
std::unique_ptr iter;
- segment->new_iterator(schema, read_opts, &iter);
+ segment->new_iterator(schema, read_opts, nullptr, &iter);
RowBlockV2 block(schema, 1024);