database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[hotfix](memtracker) Fix expired DCHECK(_limit != -1); and segment_meta_mem_tracker inelegant end (#14223)

Browse Source
This commit is contained in:
Xinyi Zou
2022-11-13 17:15:29 +08:00
committed by GitHub
parent 72748c229a
commit 3bc26f773d
4 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
be/src/olap/rowset/segment_v2/segment.cpp
View File

@ -79,7 +79,8 @@ Segment::Segment(uint32_t segment_id, RowsetId rowset_id, TabletSchemaSPtr table
Segment::~Segment() {
#ifndef BE_TEST
StorageEngine::instance()->segment_meta_mem_tracker()->release(_meta_mem_usage);
if (StorageEngine::instance())
StorageEngine::instance()->segment_meta_mem_tracker()->release(_meta_mem_usage);
#endif
}
@ -149,7 +150,8 @@ Status Segment::_parse_footer() {
_file_reader->path().native(), file_size, 12 + footer_length);
}
_meta_mem_usage += footer_length;
StorageEngine::instance()->segment_meta_mem_tracker()->consume(footer_length);
if (StorageEngine::instance())
StorageEngine::instance()->segment_meta_mem_tracker()->consume(footer_length);
std::string footer_buf;
footer_buf.resize(footer_length);
@ -214,7 +216,8 @@ Status Segment::load_index() {
DCHECK(footer.has_short_key_page_footer());
_meta_mem_usage += body.get_size();
StorageEngine::instance()->segment_meta_mem_tracker()->consume(body.get_size());
if (StorageEngine::instance())
StorageEngine::instance()->segment_meta_mem_tracker()->consume(body.get_size());
_sk_index_decoder.reset(new ShortKeyIndexDecoder);
return _sk_index_decoder->parse(body, footer.short_key_page_footer());
}

3
be/src/olap/storage_engine.cpp
View File

@ -108,7 +108,7 @@ StorageEngine::StorageEngine(const EngineOptions& options)
_is_all_cluster_id_exist(true),
_mem_tracker(std::make_shared<MemTracker>("StorageEngine")),
_segcompaction_mem_tracker(std::make_shared<MemTracker>("SegCompaction")),
_segment_meta_mem_tracker(std::make_unique<MemTracker>("SegmentMeta")),
_segment_meta_mem_tracker(std::make_shared<MemTracker>("SegmentMeta")),
_stop_background_threads_latch(1),
_tablet_manager(new TabletManager(config::tablet_map_shard_size)),
_txn_manager(new TxnManager(config::txn_map_shard_size, config::txn_shard_size)),
@ -146,6 +146,7 @@ StorageEngine::~StorageEngine() {
if (_tablet_meta_checkpoint_thread_pool) {
_tablet_meta_checkpoint_thread_pool->shutdown();
}
_s_instance = nullptr;
}
void StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) {

4
be/src/olap/storage_engine.h
View File

@ -179,7 +179,7 @@ public:
Status get_compaction_status_json(std::string* result);
MemTracker* segment_meta_mem_tracker() { return _segment_meta_mem_tracker.get(); }
std::shared_ptr<MemTracker> segment_meta_mem_tracker() { return _segment_meta_mem_tracker; }
std::shared_ptr<MemTracker> segcompaction_mem_tracker() { return _segcompaction_mem_tracker; }
// check cumulative compaction config
@ -329,7 +329,7 @@ private:
std::shared_ptr<MemTracker> _segcompaction_mem_tracker;
// This mem tracker is only for tracking memory use by segment meta data such as footer or index page.
// The memory consumed by querying is tracked in segment iterator.
std::unique_ptr<MemTracker> _segment_meta_mem_tracker;
std::shared_ptr<MemTracker> _segment_meta_mem_tracker;
CountDownLatch _stop_background_threads_latch;
scoped_refptr<Thread> _unused_rowset_monitor_thread;

1
be/src/runtime/memory/mem_tracker_limiter.cpp
View File

@ -207,7 +207,6 @@ void MemTrackerLimiter::print_log_process_usage(const std::string& msg) {
std::string MemTrackerLimiter::mem_limit_exceeded(const std::string& msg,
const std::string& limit_exceeded_errmsg) {
DCHECK(_limit != -1);
STOP_CHECK_THREAD_MEM_TRACKER_LIMIT();
std::string detail = fmt::format(
"Memory limit exceeded:<consuming tracker:<{}>, {}>, executing msg:<{}>. backend {} "