From d57c2344e1a2cac20b37e9f00a060b5027a3f3e1 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 16 Jun 2021 09:44:24 +0800 Subject: [PATCH] [MemTracker] Refactored the hierarchical structure of memtracker (#5956) To avoid showing too many memtracker on BE web pages. The MemTracker level now has 3 levels: OVERVIEW, TASK and VERBOSE. OVERVIEW Mainly used for main memory consumption module such as Query/Load/Metadata. TASK is mainly used to record the memory overhead of a single task such as a single query, load, and compaction task. VERBOSE is used for other more detailed memtrackers. --- be/src/exec/olap_scanner.cpp | 2 +- be/src/http/action/compaction_action.h | 2 +- be/src/http/default_path_handlers.cpp | 12 +++-- be/src/olap/compaction.cpp | 4 +- be/src/olap/fs/file_block_manager.cpp | 4 +- be/src/olap/lru_cache.cpp | 2 +- be/src/olap/page_cache.cpp | 2 +- be/src/olap/rowset/beta_rowset.cpp | 2 +- be/src/olap/rowset/beta_rowset_reader.cpp | 2 +- be/src/olap/rowset/segment_reader.cpp | 2 +- be/src/olap/rowset/segment_v2/segment.cpp | 22 +++++--- be/src/olap/rowset/segment_v2/segment.h | 9 ++-- .../rowset/segment_v2/segment_iterator.cpp | 6 +-- be/src/olap/schema_change.cpp | 4 +- be/src/olap/storage_engine.cpp | 8 +-- be/src/olap/storage_engine.h | 4 ++ be/src/olap/tablet_manager.cpp | 2 +- be/src/runtime/buffered_block_mgr2.cc | 2 - be/src/runtime/exec_env.h | 3 ++ be/src/runtime/exec_env_init.cpp | 4 +- be/src/runtime/load_channel.cpp | 2 +- be/src/runtime/load_channel_mgr.cpp | 2 +- be/src/runtime/mem_tracker.cpp | 4 +- be/src/runtime/mem_tracker.h | 6 +-- be/src/runtime/plan_fragment_executor.cpp | 2 +- .../olap/rowset/segment_v2/segment_test.cpp | 52 +++++++++---------- 26 files changed, 94 insertions(+), 72 deletions(-) diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 1f719c9dd6..e97d3cecf5 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -53,7 +53,7 @@ OlapScanner::OlapScanner(RuntimeState* runtime_state, OlapScanNode* parent, bool _version(-1), _mem_tracker(MemTracker::CreateTracker( runtime_state->fragment_mem_tracker()->limit(), "OlapScanner", - runtime_state->fragment_mem_tracker(), true, true, MemTrackerLevel::DEBUG)) { + runtime_state->fragment_mem_tracker(), true, true, MemTrackerLevel::VERBOSE)) { _rows_read_counter = parent->rows_read_counter(); _rows_pushed_cond_filtered_counter = parent->_rows_pushed_cond_filtered_counter; } diff --git a/be/src/http/action/compaction_action.h b/be/src/http/action/compaction_action.h index 5a7b936c4b..57513de50c 100644 --- a/be/src/http/action/compaction_action.h +++ b/be/src/http/action/compaction_action.h @@ -42,7 +42,7 @@ public: CompactionAction(CompactionActionType type) : _type(type) { _compaction_mem_tracker = type == RUN_COMPACTION ? - MemTracker::CreateTracker(-1, "ManualCompaction") : nullptr; + MemTracker::CreateTracker(-1, "ManualCompaction", nullptr, false, false, MemTrackerLevel::TASK) : nullptr; } virtual ~CompactionAction() {} diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index d11991d36f..d8416970dc 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -129,7 +129,8 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr " data-search='true' " " class='table table-striped'>\n"; (*output) << "" - "Id" + "Id" "Parent" "Limit" "Current Consumption" "Peak Consumption"; + ">Peak Consumption" + "Use Count"; (*output) << "\n"; std::vector> trackers; @@ -147,10 +150,11 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr string limit_str = tracker->limit() == -1 ? "none" : ItoaKMGT(tracker->limit()); string current_consumption_str = ItoaKMGT(tracker->consumption()); string peak_consumption_str = ItoaKMGT(tracker->peak_consumption()); + int64_t use_count = tracker.use_count(); (*output) << strings::Substitute( "$0$1$2" // id, parent, limit - "$3$4\n", // current, peak - tracker->label(), parent, limit_str, current_consumption_str, peak_consumption_str); + "$3$4$5\n", // current, peak + tracker->label(), parent, limit_str, current_consumption_str, peak_consumption_str, use_count); } (*output) << "\n"; } diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 383ae9ced1..f735be8110 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -28,7 +28,7 @@ namespace doris { Compaction::Compaction(TabletSharedPtr tablet, const std::string& label, const std::shared_ptr& parent_tracker) - : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker, true, false)), + : _mem_tracker(MemTracker::CreateTracker(-1, label, parent_tracker, true, false, MemTrackerLevel::TASK)), _readers_tracker(MemTracker::CreateTracker(-1, "CompactionReaderTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker, true, false)), _writer_tracker(MemTracker::CreateTracker(-1, "CompationWriterTracker:" + std::to_string(tablet->tablet_id()), _mem_tracker, @@ -177,7 +177,7 @@ OLAPStatus Compaction::construct_input_rowset_readers() { RETURN_NOT_OK(rowset->create_reader( MemTracker::CreateTracker( -1, "Compaction:RowsetReader:" + rowset->rowset_id().to_string(), - _readers_tracker, true, true, MemTrackerLevel::DEBUG), + _readers_tracker, true, true), &rs_reader)); _input_rs_readers.push_back(std::move(rs_reader)); } diff --git a/be/src/olap/fs/file_block_manager.cpp b/be/src/olap/fs/file_block_manager.cpp index 2f49136a01..f62f95dbaf 100644 --- a/be/src/olap/fs/file_block_manager.cpp +++ b/be/src/olap/fs/file_block_manager.cpp @@ -368,8 +368,8 @@ Status FileReadableBlock::readv(uint64_t offset, const Slice* results, size_t re FileBlockManager::FileBlockManager(Env* env, BlockManagerOptions opts) : _env(DCHECK_NOTNULL(env)), _opts(std::move(opts)), - _mem_tracker( - MemTracker::CreateTracker(-1, "FileBlockManager", _opts.parent_mem_tracker)) { + _mem_tracker(MemTracker::CreateTracker(-1, "FileBlockManager", _opts.parent_mem_tracker, + false, false, MemTrackerLevel::OVERVIEW)) { if (_opts.enable_metric) { _metrics.reset(new internal::BlockManagerMetrics()); } diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp index f420cc5927..b5919091b2 100644 --- a/be/src/olap/lru_cache.cpp +++ b/be/src/olap/lru_cache.cpp @@ -412,7 +412,7 @@ uint32_t ShardedLRUCache::_shard(uint32_t hash) { ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity, std::shared_ptr parent) : _name(name), _last_id(1), - _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false)) { + _mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false, MemTrackerLevel::OVERVIEW)) { const size_t per_shard = (total_capacity + (kNumShards - 1)) / kNumShards; for (int s = 0; s < kNumShards; s++) { _shards[s].set_capacity(per_shard); diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp index 6ce9faba6b..056ad63192 100644 --- a/be/src/olap/page_cache.cpp +++ b/be/src/olap/page_cache.cpp @@ -29,7 +29,7 @@ void StoragePageCache::create_global_cache(size_t capacity, int32_t index_cache_ StoragePageCache::StoragePageCache(size_t capacity, int32_t index_cache_percentage) : _index_cache_percentage(index_cache_percentage), - _mem_tracker(MemTracker::CreateTracker(capacity, "StorgePageCache")) { + _mem_tracker(MemTracker::CreateTracker(capacity, "StorgePageCache", nullptr, true, true, MemTrackerLevel::OVERVIEW)) { if (index_cache_percentage == 0) { _data_page_cache = std::unique_ptr(new_lru_cache("DataPageCache", capacity, _mem_tracker)); } else if (index_cache_percentage == 100) { diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index fc03a7c094..a7a7dcb357 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -50,7 +50,7 @@ OLAPStatus BetaRowset::do_load(bool /*use_cache*/, std::shared_ptr p for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { std::string seg_path = segment_file_path(_rowset_path, rowset_id(), seg_id); std::shared_ptr segment; - auto s = segment_v2::Segment::open(seg_path, seg_id, _schema, parent, &segment); + auto s = segment_v2::Segment::open(seg_path, seg_id, _schema, &segment); if (!s.ok()) { LOG(WARNING) << "failed to open segment " << seg_path << " under rowset " << unique_id() << " : " << s.to_string(); diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index d0e74966f0..ee13dad67c 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -91,7 +91,7 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) { std::vector> seg_iterators; for (auto& seg_ptr : _rowset->_segments) { std::unique_ptr iter; - auto s = seg_ptr->new_iterator(schema, read_options, &iter); + auto s = seg_ptr->new_iterator(schema, read_options, _parent_tracker, &iter); if (!s.ok()) { LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() << "]: " << s.to_string(); return OLAP_ERR_ROWSET_READER_INIT; diff --git a/be/src/olap/rowset/segment_reader.cpp b/be/src/olap/rowset/segment_reader.cpp index 38d3d1fa93..8314d0af1d 100644 --- a/be/src/olap/rowset/segment_reader.cpp +++ b/be/src/olap/rowset/segment_reader.cpp @@ -58,7 +58,7 @@ SegmentReader::SegmentReader(const std::string file, SegmentGroup* segment_group _is_using_mmap(false), _is_data_loaded(false), _buffer_size(0), - _tracker(MemTracker::CreateTracker(-1, "SegmentReader:" + file, parent_tracker)), + _tracker(MemTracker::CreateTracker(-1, "SegmentReader:" + file, parent_tracker, false)), _mem_pool(new MemPool(_tracker.get())), _shared_buffer(NULL), _lru_cache(lru_cache), diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 992d8c414d..fbfd52fb74 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -27,6 +27,7 @@ #include "olap/rowset/segment_v2/page_io.h" #include "olap/rowset/segment_v2/segment_iterator.h" #include "olap/rowset/segment_v2/segment_writer.h" // k_segment_magic_length +#include "olap/storage_engine.h" #include "olap/tablet_schema.h" #include "util/crc32c.h" #include "util/slice.h" // Slice @@ -37,16 +38,22 @@ namespace segment_v2 { using strings::Substitute; Status Segment::open(std::string filename, uint32_t segment_id, const TabletSchema* tablet_schema, - std::shared_ptr parent, std::shared_ptr* output) { - std::shared_ptr segment(new Segment(std::move(filename), segment_id, tablet_schema, std::move(parent))); + std::shared_ptr* output) { + std::shared_ptr segment(new Segment(std::move(filename), segment_id, tablet_schema)); RETURN_IF_ERROR(segment->_open()); output->swap(segment); return Status::OK(); } -Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema, std::shared_ptr parent) +Segment::Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema) : _fname(std::move(fname)), _segment_id(segment_id), - _tablet_schema(tablet_schema), _mem_tracker(MemTracker::CreateTracker(-1, "Segment", std::move(parent), false, true)) {} + _tablet_schema(tablet_schema) { +#ifndef BE_TEST + _mem_tracker = MemTracker::CreateTracker(-1, "Segment", StorageEngine::instance()->tablet_mem_tracker(), false); +#else + _mem_tracker = MemTracker::CreateTracker(-1, "Segment", nullptr, false); +#endif +} Segment::~Segment() { _mem_tracker->Release(_mem_tracker->consumption()); @@ -59,6 +66,7 @@ Status Segment::_open() { } Status Segment::new_iterator(const Schema& schema, const StorageReadOptions& read_options, + std::shared_ptr parent, std::unique_ptr* iter) { DCHECK_NOTNULL(read_options.stats); read_options.stats->total_segment_number++; @@ -80,7 +88,7 @@ Status Segment::new_iterator(const Schema& schema, const StorageReadOptions& rea } RETURN_IF_ERROR(_load_index()); - iter->reset(new SegmentIterator(this->shared_from_this(), schema, _mem_tracker)); + iter->reset(new SegmentIterator(this->shared_from_this(), schema, parent)); iter->get()->init(read_options); return Status::OK(); } @@ -188,7 +196,7 @@ Status Segment::_create_column_readers() { return Status::OK(); } -Status Segment::new_column_iterator(uint32_t cid, ColumnIterator** iter) { +Status Segment::new_column_iterator(uint32_t cid, std::shared_ptr parent, ColumnIterator** iter) { if (_column_readers[cid] == nullptr) { const TabletColumn& tablet_column = _tablet_schema->column(cid); if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) { @@ -200,7 +208,7 @@ Status Segment::new_column_iterator(uint32_t cid, ColumnIterator** iter) { tablet_column.has_default_value(), tablet_column.default_value(), tablet_column.is_nullable(), type_info, tablet_column.length())); ColumnIteratorOptions iter_opts; - iter_opts.mem_tracker = MemTracker::CreateTracker(-1, "DefaultColumnIterator", _mem_tracker, false); + iter_opts.mem_tracker = MemTracker::CreateTracker(-1, "DefaultColumnIterator", parent, false); RETURN_IF_ERROR(default_value_iter->init(iter_opts)); *iter = default_value_iter.release(); diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index cc883b19fe..0cfd40889a 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -60,18 +60,19 @@ using SegmentSharedPtr = std::shared_ptr; class Segment : public std::enable_shared_from_this { public: static Status open(std::string filename, uint32_t segment_id, const TabletSchema* tablet_schema, - std::shared_ptr parent, std::shared_ptr* output); + std::shared_ptr* output); ~Segment(); Status new_iterator(const Schema& schema, const StorageReadOptions& read_options, + std::shared_ptr parent, std::unique_ptr* iter); uint64_t id() const { return _segment_id; } uint32_t num_rows() const { return _footer.num_rows(); } - Status new_column_iterator(uint32_t cid, ColumnIterator** iter); + Status new_column_iterator(uint32_t cid, std::shared_ptr parent, ColumnIterator** iter); Status new_bitmap_index_iterator(uint32_t cid, BitmapIndexIterator** iter); @@ -104,7 +105,7 @@ public: private: DISALLOW_COPY_AND_ASSIGN(Segment); - Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema, std::shared_ptr parent); + Segment(std::string fname, uint32_t segment_id, const TabletSchema* tablet_schema); // open segment file and read the minimum amount of necessary information (footer) Status _open(); Status _parse_footer(); @@ -119,6 +120,8 @@ private: uint32_t _segment_id; const TabletSchema* _tablet_schema; + // This mem tracker is only for tracking memory use by segment meta data such as footer or index page. + // The memory consumed by querying is tracked in segment iterator. std::shared_ptr _mem_tracker; SegmentFooterPB _footer; diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 859cca50c4..50e8febb96 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -99,7 +99,7 @@ SegmentIterator::SegmentIterator(std::shared_ptr segment, const Schema& _lazy_materialization_read(false), _inited(false) { // use for count the mem use of ColumnIterator - _mem_tracker = MemTracker::CreateTracker(-1, "SegmentIterator", std::move(parent), 0); + _mem_tracker = MemTracker::CreateTracker(-1, "SegmentIterator", std::move(parent), false); } SegmentIterator::~SegmentIterator() { @@ -194,7 +194,7 @@ Status SegmentIterator::_prepare_seek(const StorageReadOptions::KeyRange& key_ra // create used column iterator for (auto cid : _seek_schema->column_ids()) { if (_column_iterators[cid] == nullptr) { - RETURN_IF_ERROR(_segment->new_column_iterator(cid, &_column_iterators[cid])); + RETURN_IF_ERROR(_segment->new_column_iterator(cid, _mem_tracker, &_column_iterators[cid])); ColumnIteratorOptions iter_opts; iter_opts.stats = _opts.stats; iter_opts.rblock = _rblock.get(); @@ -322,7 +322,7 @@ Status SegmentIterator::_init_return_column_iterators() { } for (auto cid : _schema.column_ids()) { if (_column_iterators[cid] == nullptr) { - RETURN_IF_ERROR(_segment->new_column_iterator(cid, &_column_iterators[cid])); + RETURN_IF_ERROR(_segment->new_column_iterator(cid, _mem_tracker, &_column_iterators[cid])); ColumnIteratorOptions iter_opts; iter_opts.stats = _opts.stats; iter_opts.use_page_cache = _opts.use_page_cache; diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 7d69f604a9..70b5d15c7d 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -1361,7 +1361,7 @@ bool SchemaChangeWithSorting::_external_sorting(vector& src_row } SchemaChangeHandler::SchemaChangeHandler() - : _mem_tracker(MemTracker::CreateTracker(-1, "SchemaChange")) { + : _mem_tracker(MemTracker::CreateTracker(-1, "SchemaChange", StorageEngine::instance()->schema_change_mem_tracker())) { REGISTER_HOOK_METRIC(schema_change_mem_consumption, [this]() { return _mem_tracker->consumption(); }); } @@ -1474,7 +1474,7 @@ OLAPStatus SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletRe reader_context.seek_columns = &return_columns; auto mem_tracker = MemTracker::CreateTracker(-1, "AlterTablet:" + std::to_string(base_tablet->tablet_id()) + "-" - + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false, MemTrackerLevel::DEBUG); + + std::to_string(new_tablet->tablet_id()), _mem_tracker, true, false, MemTrackerLevel::TASK); do { // get history data to be converted and it will check if there is hold in base tablet diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index e5eaf71e3f..fd20d31a0a 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -113,7 +113,9 @@ StorageEngine::StorageEngine(const EngineOptions& options) _index_stream_lru_cache(NULL), _file_cache(nullptr), _compaction_mem_tracker( - MemTracker::CreateTracker(-1, "AutoCompaction")), + MemTracker::CreateTracker(-1, "AutoCompaction", nullptr, false, false, MemTrackerLevel::OVERVIEW)), + _tablet_mem_tracker( + MemTracker::CreateTracker(-1, "TabletHeader", nullptr, false, false, MemTrackerLevel::OVERVIEW)), _stop_background_threads_latch(1), _tablet_manager(new TabletManager(config::tablet_map_shard_size)), _txn_manager(new TxnManager(config::txn_map_shard_size, config::txn_shard_size)), @@ -1028,14 +1030,14 @@ bool StorageEngine::check_rowset_id_in_unused_rowsets(const RowsetId& rowset_id) void StorageEngine::create_cumulative_compaction( TabletSharedPtr best_tablet, std::shared_ptr& cumulative_compaction) { - std::string tracker_label = "StorageEngine:CumulativeCompaction:" + std::to_string(syscall(__NR_gettid)); + std::string tracker_label = "StorageEngine:CumulativeCompaction:" + std::to_string(best_tablet->tablet_id()); cumulative_compaction.reset( new CumulativeCompaction(best_tablet, tracker_label, _compaction_mem_tracker)); } void StorageEngine::create_base_compaction(TabletSharedPtr best_tablet, std::shared_ptr& base_compaction) { - std::string tracker_label = "StorageEngine:BaseCompaction:" + std::to_string(syscall(__NR_gettid)); + std::string tracker_label = "StorageEngine:BaseCompaction:" + std::to_string(best_tablet->tablet_id()); base_compaction.reset(new BaseCompaction(best_tablet, tracker_label, _compaction_mem_tracker)); } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 8980e82b02..26afb53922 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -179,6 +179,9 @@ public: Status get_compaction_status_json(std::string* result); + std::shared_ptr tablet_mem_tracker() { return _tablet_mem_tracker; } + std::shared_ptr schema_change_mem_tracker() { return _schema_change_mem_tracker; } + private: // Instance should be inited from `static open()` // MUST NOT be called in other circumstances. @@ -306,6 +309,7 @@ private: std::unordered_map _unused_rowsets; std::shared_ptr _compaction_mem_tracker; + std::shared_ptr _tablet_mem_tracker; std::shared_ptr _schema_change_mem_tracker; CountDownLatch _stop_background_threads_latch; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index b1e7cf3429..09a9e87db7 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -71,7 +71,7 @@ static bool _cmp_tablet_by_create_time(const TabletSharedPtr& a, const TabletSha } TabletManager::TabletManager(int32_t tablet_map_lock_shard_size) - : _mem_tracker(MemTracker::CreateTracker(-1, "TabletManager", nullptr, false)), + : _mem_tracker(MemTracker::CreateTracker(-1, "TabletMeta", nullptr, false, false, MemTrackerLevel::OVERVIEW)), _tablets_shards_size(tablet_map_lock_shard_size), _tablets_shards_mask(tablet_map_lock_shard_size - 1), _last_update_stat_ms(0) { diff --git a/be/src/runtime/buffered_block_mgr2.cc b/be/src/runtime/buffered_block_mgr2.cc index 9621092500..48d4a76895 100644 --- a/be/src/runtime/buffered_block_mgr2.cc +++ b/be/src/runtime/buffered_block_mgr2.cc @@ -1288,8 +1288,6 @@ void BufferedBlockMgr2::init(DiskIoMgr* io_mgr, RuntimeProfile* parent_profile, _integrity_check_timer = ADD_TIMER(_profile.get(), "TotalIntegrityCheckTime"); // Create a new mem_tracker and allocate buffers. - // _mem_tracker.reset(new MemTracker( - // profile(), mem_limit, -1, "Block Manager", parent_tracker)); _mem_tracker = MemTracker::CreateTracker(mem_limit, "BufferedBlockMgr2", parent_tracker); _initialized = true; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index 5968a8e1a9..2fcf901124 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -140,6 +140,9 @@ public: PluginMgr* plugin_mgr() { return _plugin_mgr; } + // The root tracker should be set before calling ExecEnv::init(); + void set_root_mem_tracker(std::shared_ptr root_tracker); + private: Status _init(const std::vector& store_paths); void _destroy(); diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 7bd31a32e4..63095d57c8 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -185,8 +185,8 @@ Status ExecEnv::_init_mem_tracker() { return Status::InternalError(ss.str()); } - _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Query", MemTracker::GetRootTracker(), - false, false); + _mem_tracker = MemTracker::CreateTracker(bytes_limit, "Process", MemTracker::GetRootTracker(), + false, false, MemTrackerLevel::OVERVIEW); LOG(INFO) << "Using global memory limit: " << PrettyPrinter::print(bytes_limit, TUnit::BYTES); RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker)); diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp index d6ade09e89..43f8a883dd 100644 --- a/be/src/runtime/load_channel.cpp +++ b/be/src/runtime/load_channel.cpp @@ -27,7 +27,7 @@ LoadChannel::LoadChannel(const UniqueId& load_id, int64_t mem_limit, int64_t tim const std::shared_ptr& mem_tracker) : _load_id(load_id), _timeout_s(timeout_s) { _mem_tracker = MemTracker::CreateTracker( - mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker, true, false); + mem_limit, "LoadChannel:" + _load_id.to_string(), mem_tracker, true, false, MemTrackerLevel::TASK); // _last_updated_time should be set before being inserted to // _load_channels in load_channel_mgr, or it may be erased // immediately by gc thread. diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp index 15e2c08fa0..f0110aaa3f 100644 --- a/be/src/runtime/load_channel_mgr.cpp +++ b/be/src/runtime/load_channel_mgr.cpp @@ -85,7 +85,7 @@ LoadChannelMgr::~LoadChannelMgr() { Status LoadChannelMgr::init(int64_t process_mem_limit) { int64_t load_mem_limit = calc_process_max_load_memory(process_mem_limit); - _mem_tracker = MemTracker::CreateTracker(load_mem_limit, "Load"); + _mem_tracker = MemTracker::CreateTracker(load_mem_limit, "LoadChannelMgr", nullptr, true, false, MemTrackerLevel::OVERVIEW); REGISTER_HOOK_METRIC(load_mem_consumption, [this]() { return _mem_tracker->consumption(); }); diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp index 9177e72c46..ea0a16984a 100644 --- a/be/src/runtime/mem_tracker.cpp +++ b/be/src/runtime/mem_tracker.cpp @@ -72,7 +72,7 @@ static std::shared_ptr root_tracker; static GoogleOnceType root_tracker_once = GOOGLE_ONCE_INIT; void MemTracker::CreateRootTracker() { - root_tracker.reset(new MemTracker(-1, "Root")); + root_tracker.reset(new MemTracker(nullptr, -1, "Root", nullptr, true, MemTrackerLevel::OVERVIEW)); root_tracker->Init(); } @@ -143,7 +143,7 @@ std::shared_ptr MemTracker::CreateTracker(int64_t byte_limit, const } MemTracker::MemTracker(int64_t byte_limit, const std::string& label) - : MemTracker(nullptr, byte_limit, label, std::shared_ptr(), true, MemTrackerLevel::RELEASE) {} + : MemTracker(nullptr, byte_limit, label, std::shared_ptr(), true, MemTrackerLevel::VERBOSE) {} MemTracker::MemTracker(RuntimeProfile* profile, int64_t byte_limit, const string& label, const std::shared_ptr& parent, bool log_usage_if_zero, MemTrackerLevel level) diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h index 142e611299..b938f6e12d 100644 --- a/be/src/runtime/mem_tracker.h +++ b/be/src/runtime/mem_tracker.h @@ -42,7 +42,7 @@ enum class MemLimit { HARD, SOFT }; /// The Level use to decide whether to show it in web page /// each MemTracker have a Level equals to parent, only be set explicit -enum class MemTrackerLevel {RELEASE = 0, DEBUG}; +enum class MemTrackerLevel {OVERVIEW = 0, TASK, VERBOSE}; class ObjectPool; class MemTracker; @@ -92,11 +92,11 @@ public: static std::shared_ptr CreateTracker( int64_t byte_limit = -1, const std::string& label = std::string(), std::shared_ptr parent = std::shared_ptr(), - bool log_usage_if_zero = true, bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::RELEASE); + bool log_usage_if_zero = true, bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::VERBOSE); static std::shared_ptr CreateTracker( RuntimeProfile* profile, int64_t byte_limit, const std::string& label = std::string(), - const std::shared_ptr& parent = std::shared_ptr(), bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::RELEASE); + const std::shared_ptr& parent = std::shared_ptr(), bool reset_label_name = true, MemTrackerLevel level = MemTrackerLevel::VERBOSE); // this is used for creating an orphan mem tracker, or for unit test. // If a mem tracker has parent, it should be created by `CreateTracker()` diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp index 3bd8a1be72..2df18a5d1f 100644 --- a/be/src/runtime/plan_fragment_executor.cpp +++ b/be/src/runtime/plan_fragment_executor.cpp @@ -132,7 +132,7 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request, _mem_tracker = MemTracker::CreateTracker(bytes_limit, "PlanFragmentExecutor:" + print_id(_query_id) + ":" + print_id(params.fragment_instance_id), - _exec_env->process_mem_tracker(), true, false); + _exec_env->process_mem_tracker(), true, false, MemTrackerLevel::TASK); _runtime_state->set_fragment_mem_tracker(_mem_tracker); LOG(INFO) << "Using query memory limit: " << PrettyPrinter::print(bytes_limit, TUnit::BYTES); diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp b/be/test/olap/rowset/segment_v2/segment_test.cpp index 2efb213493..cbb3ac8c72 100644 --- a/be/test/olap/rowset/segment_v2/segment_test.cpp +++ b/be/test/olap/rowset/segment_v2/segment_test.cpp @@ -136,7 +136,7 @@ protected: ASSERT_TRUE(st.ok()); ASSERT_TRUE(wblock->close().ok()); - st = Segment::open(filename, 0, &query_schema, MemTracker::CreateTracker(-1, "Segment", nullptr, false), res); + st = Segment::open(filename, 0, &query_schema, res); ASSERT_TRUE(st.ok()); ASSERT_EQ(nrows, (*res)->num_rows()); } @@ -164,7 +164,7 @@ TEST_F(SegmentReaderWriterTest, normal) { StorageReadOptions read_opts; read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -220,7 +220,7 @@ TEST_F(SegmentReaderWriterTest, normal) { read_opts.stats = &stats; read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), true); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 100); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -246,7 +246,7 @@ TEST_F(SegmentReaderWriterTest, normal) { read_opts.stats = &stats; read_opts.key_ranges.emplace_back(lower_bound.get(), false, nullptr, false); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 100); ASSERT_TRUE(iter->next_batch(&block).is_end_of_file()); @@ -275,7 +275,7 @@ TEST_F(SegmentReaderWriterTest, normal) { read_opts.stats = &stats; read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), false); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 100); ASSERT_TRUE(iter->next_batch(&block).is_end_of_file()); @@ -312,7 +312,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { read_opts.stats = &stats; std::unique_ptr iter; - ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); + ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok()); RowBlockV2 block(read_schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -336,7 +336,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { read_opts.stats = &stats; std::unique_ptr iter; - ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); + ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok()); RowBlockV2 block(read_schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -356,7 +356,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { read_opts.stats = &stats; std::unique_ptr iter; - ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); + ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok()); RowBlockV2 block(read_schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -388,7 +388,7 @@ TEST_F(SegmentReaderWriterTest, LazyMaterialization) { read_opts.stats = &stats; std::unique_ptr iter; - ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, &iter).ok()); + ASSERT_TRUE(segment->new_iterator(read_schema, read_opts, nullptr, &iter).ok()); RowBlockV2 block(read_schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -450,7 +450,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { read_opts.conditions = conditions.get(); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1); @@ -473,7 +473,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { read_opts.conditions = conditions.get(); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -533,7 +533,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { read_opts.delete_conditions.push_back(delete_conditions.get()); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -581,7 +581,7 @@ TEST_F(SegmentReaderWriterTest, TestIndex) { ASSERT_EQ(OLAP_SUCCESS, conditions->append_condition(condition)); read_opts.conditions = conditions.get(); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); ASSERT_TRUE(iter->next_batch(&block).is_end_of_file()); @@ -674,7 +674,7 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) { StorageReadOptions read_opts; read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -724,7 +724,7 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) { StorageReadOptions read_opts; read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -817,7 +817,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { { std::shared_ptr segment; - st = Segment::open(fname, 0, tablet_schema.get(), MemTracker::CreateTracker(-1, "Segment", nullptr, false), &segment); + st = Segment::open(fname, 0, tablet_schema.get(), &segment); ASSERT_TRUE(st.ok()); ASSERT_EQ(4096, segment->num_rows()); Schema schema(*tablet_schema); @@ -827,7 +827,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { StorageReadOptions read_opts; read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -880,7 +880,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { read_opts.stats = &stats; read_opts.key_ranges.emplace_back(lower_bound.get(), false, nullptr, false); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 100); st = iter->next_batch(&block); @@ -913,7 +913,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { read_opts.stats = &stats; read_opts.key_ranges.emplace_back(lower_bound.get(), false, upper_bound.get(), false); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 100); st = iter->next_batch(&block); @@ -937,7 +937,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { read_opts.conditions = conditions.get(); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); int left = 4 * 1024; @@ -994,7 +994,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { read_opts.conditions = conditions.get(); std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); @@ -1033,7 +1033,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -1055,7 +1055,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -1077,7 +1077,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); ASSERT_FALSE(iter->next_batch(&block).ok()); @@ -1101,7 +1101,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024); ASSERT_TRUE(iter->next_batch(&block).ok()); @@ -1124,7 +1124,7 @@ TEST_F(SegmentReaderWriterTest, TestBitmapPredicate) { read_opts.stats = &stats; std::unique_ptr iter; - segment->new_iterator(schema, read_opts, &iter); + segment->new_iterator(schema, read_opts, nullptr, &iter); RowBlockV2 block(schema, 1024);