diff --git a/deps/oblib/src/lib/CMakeLists.txt b/deps/oblib/src/lib/CMakeLists.txt index 685d5183b..e87fbe14a 100644 --- a/deps/oblib/src/lib/CMakeLists.txt +++ b/deps/oblib/src/lib/CMakeLists.txt @@ -246,6 +246,7 @@ ob_set_subtarget(ob_malloc_object_list common_alloc alloc/alloc_struct.cpp alloc/block_set.cpp alloc/memory_dump.cpp + alloc/ob_free_log_printer.cpp alloc/ob_malloc_allocator.cpp alloc/ob_malloc_callback.cpp alloc/ob_malloc_sample_struct.cpp diff --git a/deps/oblib/src/lib/alloc/alloc_interface.h b/deps/oblib/src/lib/alloc/alloc_interface.h index 9bb729425..3e32ac2ae 100644 --- a/deps/oblib/src/lib/alloc/alloc_interface.h +++ b/deps/oblib/src/lib/alloc/alloc_interface.h @@ -36,6 +36,11 @@ public: virtual ABlock *alloc_block(uint64_t size, const ObMemAttr &attr) = 0; virtual void free_block(ABlock *block) = 0; virtual int64_t sync_wash(int64_t wash_size) = 0; + virtual int64_t get_tenant_id() { return tenant_id_; } + virtual int64_t get_ctx_id() { return ctx_id_; } + void set_tenant_id(const int64_t tenant_id) { tenant_id_ = tenant_id; } + void set_ctx_id(const int64_t ctx_id) { ctx_id_ = ctx_id; } +protected: int64_t tenant_id_; int64_t ctx_id_; }; // end of class IBlockMgr diff --git a/deps/oblib/src/lib/alloc/memory_dump.cpp b/deps/oblib/src/lib/alloc/memory_dump.cpp index 0ae567089..a176a6ccc 100644 --- a/deps/oblib/src/lib/alloc/memory_dump.cpp +++ b/deps/oblib/src/lib/alloc/memory_dump.cpp @@ -14,6 +14,7 @@ #include "lib/alloc/memory_dump.h" #include +#include "lib/alloc/ob_free_log_printer.h" #include "lib/signal/ob_signal_struct.h" #include "lib/rc/context.h" #include "lib/utility/utility.h" @@ -428,13 +429,7 @@ int malloc_sample_stat(uint64_t tenant_id, uint64_t ctx_id, ObMallocSampleKey key; key.tenant_id_ = tenant_id; key.ctx_id_ = ctx_id; - void **backtrace = reinterpret_cast(&object->data_[offset]); - int32_t bt_size = 0; - while (bt_size < AOBJECT_BACKTRACE_COUNT && nullptr != backtrace[bt_size]) { - key.bt_[bt_size] = backtrace[bt_size]; - ++bt_size; - } - key.bt_size_ = bt_size; + MEMCPY((char*)key.bt_, &object->data_[offset], AOBJECT_BACKTRACE_SIZE); STRNCPY(key.label_, object->label_, sizeof(key.label_)); key.label_[sizeof(key.label_) - 1] = '\0'; ObMallocSampleValue *item = malloc_sample_map.get(key); @@ -581,6 +576,7 @@ void ObMemoryDump::handle(void *task) ObLatchWGuard guard(iter_lock_, common::ObLatchIds::MEM_DUMP_ITER_LOCK); std::swap(r_stat_, w_stat_); } + ObFreeLogPrinter::get_instance().disable_free_log(); } else { int fd = -1; if (-1 == (fd = ::open(LOG_FILE, diff --git a/deps/oblib/src/lib/alloc/ob_free_log_printer.cpp b/deps/oblib/src/lib/alloc/ob_free_log_printer.cpp new file mode 100644 index 000000000..020c3f189 --- /dev/null +++ b/deps/oblib/src/lib/alloc/ob_free_log_printer.cpp @@ -0,0 +1,118 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/alloc/ob_free_log_printer.h" +#include "lib/alloc/alloc_struct.h" +#include "lib/alloc/ob_malloc_sample_struct.h" +#include "lib/time/ob_time_utility.h" +#include "lib/utility/utility.h" +using namespace oceanbase::lib; +using namespace oceanbase::common; + +ObFreeLogPrinter::ObFreeLogPrinter() : level_(DEFAULT), lock_() +{} + +ObFreeLogPrinter& ObFreeLogPrinter::get_instance() +{ + static ObFreeLogPrinter free_log_print; + return free_log_print; +} + +int ObFreeLogPrinter::get_level() +{ + int ret = DEFAULT; + int reason = g_alloc_failed_ctx().reason_; + switch (reason) { + case CTX_HOLD_REACH_LIMIT : { + ret = CTX; + break; + } + case TENANT_HOLD_REACH_LIMIT: { + ret = TENANT; + break; + } + case SERVER_HOLD_REACH_LIMIT: { + ret = SERVER; + break; + } + case PHYSICAL_MEMORY_EXHAUST: { + ret = SERVER; + break; + } + } + return ret; +} + +void ObFreeLogPrinter::enable_free_log(int64_t tenant_id, int64_t ctx_id, int level) +{ + bool has_lock = false; + if (DEFAULT == level_ && OB_SUCCESS == lock_.trylock()) { + DEFER(lock_.unlock()); + last_enable_time_ = ObTimeUtility::current_time(); + tenant_id_ = tenant_id; + ctx_id_ = ctx_id; + level_ = level; + has_lock = true; + } + if (has_lock) { + _OB_LOG(INFO, "start to print free log"); + } +} + +void ObFreeLogPrinter::disable_free_log() +{ + bool has_lock = false; + if (DEFAULT != level_ && OB_SUCCESS == lock_.trylock()) { + DEFER(lock_.unlock()); + level_ = DEFAULT; + has_lock = true; + } + if (has_lock) { + _OB_LOG(INFO, "finish to print free log"); + } +} + +void ObFreeLogPrinter::print_free_log(int64_t tenant_id, int64_t ctx_id, AObject *obj) +{ + if (DEFAULT != level_ && obj->on_malloc_sample_) { + if (ObTimeUtility::current_time() - last_enable_time_ > MAX_FREE_LOG_TIME) { + disable_free_log(); + } else { + bool allowed = false; + switch (level_) { + case CTX: { + allowed = tenant_id == tenant_id_ && ctx_id == ctx_id_; + break; + } + case TENANT: { + allowed = tenant_id == tenant_id_; + break; + } + case SERVER: { + allowed = true; + break; + } + } + if (allowed) { + char buf[MAX_BACKTRACE_LENGTH]; + int64_t addrs[AOBJECT_BACKTRACE_COUNT]; + int64_t offset = obj->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE; + MEMCPY((char*)addrs, &obj->data_[offset], sizeof(addrs)); + IGNORE_RETURN parray(buf, sizeof(buf), addrs, ARRAYSIZEOF(addrs)); + allow_next_syslog(); + _OB_LOG(INFO, "[MEMORY][FREE_LOG] tenant_id=%ld, ctx_name=%s, mod=%s, alloc_bytes=%u, malloc_bt=%s\n", + tenant_id, get_global_ctx_info().get_ctx_name(ctx_id), + obj->label_, obj->alloc_bytes_, buf); + } + } + } +} \ No newline at end of file diff --git a/deps/oblib/src/lib/alloc/ob_free_log_printer.h b/deps/oblib/src/lib/alloc/ob_free_log_printer.h new file mode 100644 index 000000000..8c963c794 --- /dev/null +++ b/deps/oblib/src/lib/alloc/ob_free_log_printer.h @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef _OB_FREE_LOG_PRINTER_H_ +#define _OB_FREE_LOG_PRINTER_H_ +#include "lib/lock/ob_mutex.h" +namespace oceanbase +{ +namespace lib +{ +class AObject; +class ObFreeLogPrinter +{ +public: + const int64_t MAX_FREE_LOG_TIME = 30 * 1000 * 1000; //30s + enum Level + { + DEFAULT = 0, + CTX = 1, + TENANT = 2, + SERVER = 3, + }; + ObFreeLogPrinter(); + static ObFreeLogPrinter& get_instance(); + static int get_level(); + void enable_free_log(int64_t tenant_id , int64_t ctx_id, int level); + void disable_free_log(); + void print_free_log(int64_t tenant_id, int64_t ctx_id, AObject *obj); +private: + int64_t last_enable_time_; + int64_t tenant_id_; + int64_t ctx_id_; + int level_; + ObMutex lock_; +}; +} +} + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h b/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h index ecfe26779..448bbc67c 100644 --- a/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h +++ b/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h @@ -22,6 +22,7 @@ namespace lib { static const int32_t AOBJECT_BACKTRACE_COUNT = 16; static const int32_t AOBJECT_BACKTRACE_SIZE = sizeof(void*) * AOBJECT_BACKTRACE_COUNT; +static const int32_t MAX_BACKTRACE_LENGTH = 512; static const int32_t MAX_MALLOC_SAMPLER_NUM = (1<<15) - 1; class ObMallocSampleLimiter @@ -51,7 +52,6 @@ struct ObMallocSampleKey bool operator==(const ObMallocSampleKey &other) const; int64_t tenant_id_; int64_t ctx_id_; - int32_t bt_size_; char label_[lib::AOBJECT_LABEL_SIZE + 1]; void *bt_[AOBJECT_BACKTRACE_COUNT]; }; @@ -140,7 +140,7 @@ inline int64_t ObMallocSampleKey::hash() const hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); hash_val = murmurhash(&ctx_id_, sizeof(ctx_id_), hash_val); hash_val = murmurhash(label_, sizeof(label_), hash_val); - hash_val = murmurhash(bt_, bt_size_ * sizeof(void*), hash_val); + hash_val = murmurhash(bt_, sizeof(bt_), hash_val); return hash_val; } @@ -148,28 +148,17 @@ inline bool ObMallocSampleKey::operator==(const ObMallocSampleKey &other) const { bool ret = true; if (tenant_id_ != other.tenant_id_ || ctx_id_ != other.ctx_id_ - || 0 != STRNCMP(label_, other.label_, sizeof(label_))) { + || 0 != STRNCMP(label_, other.label_, sizeof(label_)) + || 0 != MEMCMP((char*)bt_, (char*)other.bt_, sizeof(bt_))) { ret = false; } - if (ret) { - if (other.bt_size_ != bt_size_) { - ret = false; - } else { - for (int i = 0; i < bt_size_; ++i) { - if ((int64_t)bt_[i] != (int64_t)other.bt_[i]) { - ret = false; - break; - } - } - } - } return ret; } #define ob_malloc_sample_backtrace(obj, size) \ { \ if (OB_UNLIKELY(obj->on_malloc_sample_)) { \ - void *addrs[100] = {nullptr}; \ + void *addrs[100]; \ int bt_len = backtrace(addrs, ARRAYSIZEOF(addrs)); \ MEMCPY(&obj->data_[size], (char*)addrs, AOBJECT_BACKTRACE_SIZE); \ if (AOBJECT_BACKTRACE_COUNT > bt_len) { \ @@ -177,7 +166,6 @@ inline bool ObMallocSampleKey::operator==(const ObMallocSampleKey &other) const } \ } \ } - } // end of namespace lib } // end of namespace oceanbase diff --git a/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp b/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp index 7a3fc41b9..95a731403 100644 --- a/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp +++ b/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp @@ -14,6 +14,7 @@ #include "lib/alloc/ob_tenant_ctx_allocator.h" #include "lib/alloc/ob_malloc_sample_struct.h" +#include "lib/alloc/ob_free_log_printer.h" #include "lib/allocator/ob_mem_leak_checker.h" #include "lib/allocator/ob_tc_malloc.h" #include "lib/utility/ob_print_utils.h" @@ -418,6 +419,9 @@ void* ObTenantCtxAllocator::common_alloc(const int64_t size, const ObMemAttr &at alloc_size - size + sizeof(AOBJECT_TAIL_MAGIC_CODE)); } if (OB_UNLIKELY(nullptr == obj) && REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + int level = ObFreeLogPrinter::get_level(); + ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_, + attr.ctx_id_, level); const char *msg = alloc_failed_msg(); LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg)); _OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, " @@ -425,6 +429,7 @@ void* ObTenantCtxAllocator::common_alloc(const int64_t size, const ObMemAttr &at attr.tenant_id_, attr.ctx_id_, get_global_ctx_info().get_ctx_name(attr.ctx_id_), ta.get_hold(), ta.get_limit(), ta.get_tenant_hold(), ta.get_tenant_limit()); + ObMallocAllocator::get_instance()->print_tenant_memory_usage(attr.tenant_id_); // 49 is the user defined signal to dump memory raise(49); } @@ -465,6 +470,9 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size, SANITY_POISON((void*)upper_align((int64_t)obj->data_ + size, 8), alloc_size - size + sizeof(AOBJECT_TAIL_MAGIC_CODE)); } else if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + int level = ObFreeLogPrinter::get_level(); + ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_, + attr.ctx_id_, level); const char *msg = alloc_failed_msg(); LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg)); _OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, " @@ -472,6 +480,7 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size, attr.tenant_id_, attr.ctx_id_, get_global_ctx_info().get_ctx_name(attr.ctx_id_), ta.get_hold(), ta.get_limit(), ta.get_tenant_hold(), ta.get_tenant_limit()); + ObMallocAllocator::get_instance()->print_tenant_memory_usage(attr.tenant_id_); // 49 is the user defined signal to dump memory raise(49); } @@ -497,6 +506,10 @@ void ObTenantCtxAllocator::common_free(void *ptr) abort_unless(block->obj_set_ != NULL); ObjectSet *os = block->obj_set_; + auto blk_mgr = os->get_block_mgr(); + int64_t tenant_id = blk_mgr->get_tenant_id(); + int64_t ctx_id = blk_mgr->get_ctx_id(); + ObFreeLogPrinter::get_instance().print_free_log(tenant_id, ctx_id, obj); os->free_object(obj); } } diff --git a/deps/oblib/src/lib/alloc/object_set.cpp b/deps/oblib/src/lib/alloc/object_set.cpp index 8887910b2..0121cacd9 100644 --- a/deps/oblib/src/lib/alloc/object_set.cpp +++ b/deps/oblib/src/lib/alloc/object_set.cpp @@ -56,7 +56,7 @@ AObject *ObjectSet::alloc_object( const uint64_t adj_size = MAX(size, MIN_AOBJECT_SIZE); const uint64_t all_size = align_up2(adj_size + AOBJECT_META_SIZE, 16); - const int64_t ctx_id = blk_mgr_->ctx_id_; + const int64_t ctx_id = blk_mgr_->get_ctx_id(); abort_unless(ctx_id == attr.ctx_id_); if (OB_UNLIKELY(common::ObCtxIds::LIBEASY == ctx_id)) { do_free_dirty_list(); @@ -279,8 +279,8 @@ void ObjectSet::free_normal_object(AObject *obj) normal_used_bytes_ -= obj->nobjs_ * AOBJECT_CELL_BYTES; AObject *newobj = merge_obj(obj); - auto ctx_id = blk_mgr_->ctx_id_; - auto tenant_id = blk_mgr_->tenant_id_; + auto ctx_id = blk_mgr_->get_ctx_id(); + auto tenant_id = blk_mgr_->get_tenant_id(); if (newobj->nobjs_ == cells_per_block_) { hold_bytes_ -= ablock_size_; normal_hold_bytes_ -= ablock_size_; @@ -376,7 +376,7 @@ void ObjectSet::free_object(AObject *obj) memset(obj->data_, 0xAA, obj->alloc_bytes_); } #endif - const int64_t ctx_id = blk_mgr_->ctx_id_; + const int64_t ctx_id = blk_mgr_->get_ctx_id(); ObDisableDiagnoseGuard diagnose_disable_guard; if (ctx_id == common::ObCtxIds::LIBEASY) { if (locker_->trylock()) { @@ -560,8 +560,8 @@ bool ObjectSet::build_free_lists() { abort_unless(NULL == bm_ && NULL == free_lists_); ObMemAttr attr; - attr.tenant_id_ = blk_mgr_->tenant_id_; - attr.ctx_id_ = blk_mgr_->ctx_id_; + attr.tenant_id_ = blk_mgr_->get_tenant_id(); + attr.ctx_id_ = blk_mgr_->get_ctx_id(); attr.label_ = common::ObModIds::OB_OBJ_FREELISTS; ABlock *new_block = alloc_block(sizeof (FreeList) * (cells_per_block_ + 1) + sizeof (BitMap) + BitMap::buf_len(cells_per_block_ + 1), attr); diff --git a/deps/oblib/src/lib/allocator/ob_allocator_v2.h b/deps/oblib/src/lib/allocator/ob_allocator_v2.h index 7b79bd04a..bd338ae88 100644 --- a/deps/oblib/src/lib/allocator/ob_allocator_v2.h +++ b/deps/oblib/src/lib/allocator/ob_allocator_v2.h @@ -147,8 +147,8 @@ inline int ObAllocator::init() blk_mgr = pm; pm_ = pm; } else { - blk_mgr_.tenant_id_ = attr_.tenant_id_; - blk_mgr_.ctx_id_ = attr_.ctx_id_; + blk_mgr_.set_tenant_id(attr_.tenant_id_); + blk_mgr_.set_ctx_id(attr_.ctx_id_); blk_mgr = &blk_mgr_; } if (OB_SUCC(ret)) { diff --git a/deps/oblib/src/lib/allocator/ob_page_manager.h b/deps/oblib/src/lib/allocator/ob_page_manager.h index 3c5f67754..99f69bac2 100644 --- a/deps/oblib/src/lib/allocator/ob_page_manager.h +++ b/deps/oblib/src/lib/allocator/ob_page_manager.h @@ -64,14 +64,12 @@ public: return tenant_id_ < tenant_id || (tenant_id_ == tenant_id && id_ < id); } - int set_tenant_ctx(const uint64_t tenant_id, const uint64_t ctx_id); + int set_tenant_ctx(const int64_t tenant_id, const int64_t ctx_id); void set_max_chunk_cache_cnt(const int cnt) { bs_.set_max_chunk_cache_cnt(cnt); } void reset(); int64_t get_hold() const; int64_t get_tid() const { return tid_; } - int64_t get_tenant_id() const { return tenant_id_; } - int64_t get_ctx_id() const { return ctx_id_; } // IBlockMgr interface virtual ABlock *alloc_block(uint64_t size, const ObMemAttr &attr=default_memattr) override; virtual void free_block(ABlock *block) override; @@ -135,7 +133,7 @@ inline ObPageManager::~ObPageManager() } } -inline int ObPageManager::set_tenant_ctx(const uint64_t tenant_id, const uint64_t ctx_id) +inline int ObPageManager::set_tenant_ctx(const int64_t tenant_id, const int64_t ctx_id) { int ret = OB_SUCCESS; auto &pmc = ObPageManagerCenter::get_instance(); diff --git a/src/diagnose/lua/ob_lua_api.cpp b/src/diagnose/lua/ob_lua_api.cpp index 396aa6626..255a11662 100644 --- a/src/diagnose/lua/ob_lua_api.cpp +++ b/src/diagnose/lua/ob_lua_api.cpp @@ -1964,8 +1964,8 @@ int select_malloc_sample_info(lua_State *L) gen.next_column(it->first.label_); // back_trace { - char bt[512]; - parray(bt, sizeof(bt), (int64_t*)*&(it->first.bt_), it->first.bt_size_); + char bt[MAX_BACKTRACE_LENGTH]; + IGNORE_RETURN parray(bt, sizeof(bt), (int64_t*)it->first.bt_, AOBJECT_BACKTRACE_COUNT); gen.next_column(bt); } // ctx_name diff --git a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp index 3140b68d8..3bba257e4 100644 --- a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp @@ -118,7 +118,7 @@ int ObMallocSampleInfo::fill_row(ObNewRow *&row) break; } case BACKTRACE: { - parray(bt_, sizeof(bt_), (int64_t*)*&(it_->first.bt_), it_->first.bt_size_); + IGNORE_RETURN parray(bt_, sizeof(bt_), (int64_t*)it_->first.bt_, AOBJECT_BACKTRACE_COUNT); cells[i].set_varchar(bt_); cells[i].set_collation_type( ObCharset::get_default_collation(ObCharset::get_default_charset())); diff --git a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h index d66cffab5..60f72bca3 100644 --- a/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h +++ b/src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h @@ -45,7 +45,7 @@ private: ALLOC_BYTES, }; char ip_buf_[common::OB_IP_STR_BUFF]; - char bt_[512]; + char bt_[lib::MAX_BACKTRACE_LENGTH]; lib::ObMallocSampleMap::const_iterator it_; lib::ObMallocSampleMap malloc_sample_map_; int64_t col_count_;