From 3279d19b7177cacdf757402533d5f27d99922c74 Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Thu, 5 Jan 2023 09:16:00 +0000 Subject: [PATCH] Print leak slice when ObSliceAlloc finds memory leak --- deps/oblib/src/lib/CMakeLists.txt | 1 + .../src/lib/allocator/ob_slice_alloc.cpp | 77 +++++++++++++++++++ deps/oblib/src/lib/allocator/ob_slice_alloc.h | 43 ++++------- mittest/mtlenv/test_tx_data_table.cpp | 39 +++++++++- 4 files changed, 132 insertions(+), 28 deletions(-) create mode 100644 deps/oblib/src/lib/allocator/ob_slice_alloc.cpp diff --git a/deps/oblib/src/lib/CMakeLists.txt b/deps/oblib/src/lib/CMakeLists.txt index ab01c8817..5cdc601d0 100644 --- a/deps/oblib/src/lib/CMakeLists.txt +++ b/deps/oblib/src/lib/CMakeLists.txt @@ -269,6 +269,7 @@ ob_set_subtarget(ob_malloc_object_list common_alloc allocator/ob_mem_leak_checker.cpp allocator/ob_mod_define.cpp allocator/ob_page_manager.cpp + allocator/ob_slice_alloc.cpp allocator/ob_tc_malloc.cpp ) ob_add_new_object_target(ob_malloc_object ob_malloc_object_list) diff --git a/deps/oblib/src/lib/allocator/ob_slice_alloc.cpp b/deps/oblib/src/lib/allocator/ob_slice_alloc.cpp new file mode 100644 index 000000000..6eea34864 --- /dev/null +++ b/deps/oblib/src/lib/allocator/ob_slice_alloc.cpp @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "ob_slice_alloc.h" + +#include "lib/container/ob_se_array.h" + +namespace oceanbase { +namespace common { + +void ObBlockSlicer::print_leak_slice() { + if (OB_ISNULL(slice_alloc_)) { + LIB_LOG(WARN, "invalid slice allocator", KP(this)); + return; + } + + int32_t limit = slice_alloc_->get_bsize(); + int32_t slice_size = slice_alloc_->get_isize(); + int64_t isize = lib::align_up2((int32_t)sizeof(Item) + slice_size, 16); + int64_t total = (limit - (int32_t)sizeof(*this)) / (isize + (int32_t)sizeof(void *)); + char *istart = (char *)lib::align_up2((uint64_t)((char *)(this + 1) + sizeof(void *) * total), 16); + if (istart + isize * total > ((char *)this) + limit) { + total--; + } + + for (int32_t i = 0; i < total; i++) { + Item *item = (Item *)(istart + i * isize); + void *slice = (void *)(item + 1); + if (flist_.is_in_queue(item)) { + // this item has been freed + } else { + LIB_LOG(WARN, "leak info : ", KP(item), KP(slice)); + } + } +} + +void ObSliceAlloc::destroy() +{ + for (int i = MAX_ARENA_NUM - 1; i >= 0; i--) { + Arena &arena = arena_[i]; + Block *old_blk = arena.clear(); + if (NULL != old_blk) { + blk_ref_[ObBlockSlicer::hash((uint64_t)old_blk) % MAX_REF_NUM].sync(); + if (old_blk->release()) { + blk_list_.add(&old_blk->dlink_); + } + } + } + ObDLink *dlink = nullptr; + dlink = blk_list_.top(); + while (OB_NOT_NULL(dlink)) { + Block *blk = CONTAINER_OF(dlink, Block, dlink_); + if (blk->recycle()) { + destroy_block(blk); + dlink = blk_list_.top(); + } else { + blk->print_leak_slice(); + _LIB_LOG( + ERROR, "there was memory leak, stock=%d, total=%d, remain=%d", blk->stock(), blk->total(), blk->remain()); + dlink = nullptr; // break + } + } + tmallocator_ = NULL; + bsize_ = 0; +} + +} // namespace common +} // namespace oceanbase \ No newline at end of file diff --git a/deps/oblib/src/lib/allocator/ob_slice_alloc.h b/deps/oblib/src/lib/allocator/ob_slice_alloc.h index ffff19b5b..10726461f 100644 --- a/deps/oblib/src/lib/allocator/ob_slice_alloc.h +++ b/deps/oblib/src/lib/allocator/ob_slice_alloc.h @@ -105,6 +105,17 @@ public: } return p; } + bool is_in_queue(void *item) { + bool is_in_queue = false; + for (uint64_t i = pop_; i < push_; i++) { + void *p = data_[i % capacity_]; + if (p == item) { + is_in_queue = true; + break; + } + } + return is_in_queue; + } private: uint64_t push_ CACHE_ALIGNED; uint64_t pop_ CACHE_ALIGNED; @@ -239,6 +250,7 @@ public: } void* get_tmallocator() { return tmallocator_; } void* get_slice_alloc() { return slice_alloc_; } + void print_leak_slice(); private: ObSliceAlloc* slice_alloc_; void* tmallocator_; @@ -280,33 +292,7 @@ public: new(this)ObSliceAlloc(size, attr, block_size, block_alloc, NULL); return ret; } - void destroy() { - for(int i = MAX_ARENA_NUM - 1; i >= 0; i--) { - Arena& arena = arena_[i]; - Block* old_blk = arena.clear(); - if (NULL != old_blk) { - blk_ref_[ObBlockSlicer::hash((uint64_t)old_blk) % MAX_REF_NUM].sync(); - if (old_blk->release()) { - blk_list_.add(&old_blk->dlink_); - } - } - } - ObDLink* dlink = nullptr; - dlink = blk_list_.top(); - while (OB_NOT_NULL(dlink)) { - Block* blk = CONTAINER_OF(dlink, Block, dlink_); - if (blk->recycle()) { - destroy_block(blk); - dlink = blk_list_.top(); - } else { - _LIB_LOG(ERROR, "there was memory leak, stock=%d, total=%d, remain=%d" - , blk->stock(), blk->total(), blk->remain()); - dlink = nullptr; // break - } - } - tmallocator_ = NULL; - bsize_ = 0; - } + void destroy(); void set_nway(int nway) { if (nway <= 0) { nway = 1; @@ -413,6 +399,9 @@ public: return snprintf(buf, limit, "SliceAlloc: nway=%d bsize/isize=%d/%d limit=%d attr=%s", nway_, bsize_, isize_, slice_limit_, to_cstring(attr_)); } + int32_t get_bsize() { return bsize_; } + int32_t get_isize() { return isize_; } + private: void release_block(Block* blk) { if (blk->release()) { diff --git a/mittest/mtlenv/test_tx_data_table.cpp b/mittest/mtlenv/test_tx_data_table.cpp index 32ad9ac98..5f29c8309 100644 --- a/mittest/mtlenv/test_tx_data_table.cpp +++ b/mittest/mtlenv/test_tx_data_table.cpp @@ -168,6 +168,8 @@ public: void do_multiple_init_iterator_test(); + void do_print_leak_slice_test(); + private: void insert_tx_data_(); @@ -801,6 +803,40 @@ void TestTxDataTable::fake_ls_(ObLS &ls) ls.ls_meta_.rebuild_seq_ = 0; } +void TestTxDataTable::do_print_leak_slice_test() +{ + const int32_t CONCURRENCY = 4; + ObMemAttr attr; + ObSliceAlloc slice_allocator; + + slice_allocator.init(128, OB_MALLOC_NORMAL_BLOCK_SIZE, default_blk_alloc, attr); + slice_allocator.set_nway(CONCURRENCY); + std::vector alloc_threads; + + for (int i = 0; i < CONCURRENCY; i++) { + alloc_threads.push_back(std::thread([&slice_allocator](){ + int64_t alloc_times = 1123; + std::vector allocated_mem_ptr; + while (--alloc_times > 0) { + void *ret = slice_allocator.alloc(); + if (nullptr != ret) { + allocated_mem_ptr.push_back(ret); + } + } + + STORAGE_LOG(INFO, "unfreed slice", KP(allocated_mem_ptr[0])); + for (int k = 1; k < allocated_mem_ptr.size(); k++) { + slice_allocator.free(allocated_mem_ptr[k]); + } + })); + } + + for (int i = 0; i < CONCURRENCY; i++) { + alloc_threads[i].join(); + } + + slice_allocator.destroy(); +} TEST_F(TestTxDataTable, basic_test) { @@ -814,8 +850,9 @@ TEST_F(TestTxDataTable, undo_status_test) { do_undo_status_test(); } TEST_F(TestTxDataTable, serialize_test) { do_tx_data_serialize_test(); } -// TEST_F(TestTxDataTable, iterate_init_test) { do_multiple_init_iterator_test(); } +// TEST_F(TestTxDataTable, print_leak_slice) { do_print_leak_slice_test(); } +// TEST_F(TestTxDataTable, iterate_init_test) { do_multiple_init_iterator_test(); } } // namespace storage