hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

checker_info to find the fast expansion mod

Browse Source
This commit is contained in:
obdev 2023-04-11 13:00:31 +08:00 committed by ob-robot
parent 2b16e9b079
commit 31a2b999d2
13 changed files with 207 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
deps/oblib/src/lib/CMakeLists.txt vendored
View File

@ -246,6 +246,7 @@ ob_set_subtarget(ob_malloc_object_list common_alloc
alloc/alloc_struct.cpp
alloc/block_set.cpp
alloc/memory_dump.cpp
alloc/ob_free_log_printer.cpp
alloc/ob_malloc_allocator.cpp
alloc/ob_malloc_callback.cpp
alloc/ob_malloc_sample_struct.cpp

5
deps/oblib/src/lib/alloc/alloc_interface.h vendored
View File

@ -36,6 +36,11 @@ public:
virtual ABlock *alloc_block(uint64_t size, const ObMemAttr &attr) = 0;
virtual void free_block(ABlock *block) = 0;
virtual int64_t sync_wash(int64_t wash_size) = 0;
virtual int64_t get_tenant_id() { return tenant_id_; }
virtual int64_t get_ctx_id() { return ctx_id_; }
void set_tenant_id(const int64_t tenant_id) { tenant_id_ = tenant_id; }
void set_ctx_id(const int64_t ctx_id) { ctx_id_ = ctx_id; }
protected:
int64_t tenant_id_;
int64_t ctx_id_;
}; // end of class IBlockMgr

10
deps/oblib/src/lib/alloc/memory_dump.cpp vendored
View File

@ -14,6 +14,7 @@
#include "lib/alloc/memory_dump.h"
#include <setjmp.h>
#include "lib/alloc/ob_free_log_printer.h"
#include "lib/signal/ob_signal_struct.h"
#include "lib/rc/context.h"
#include "lib/utility/utility.h"
@ -428,13 +429,7 @@ int malloc_sample_stat(uint64_t tenant_id, uint64_t ctx_id,
ObMallocSampleKey key;
key.tenant_id_ = tenant_id;
key.ctx_id_ = ctx_id;
void **backtrace = reinterpret_cast<void**>(&object->data_[offset]);
int32_t bt_size = 0;
while (bt_size < AOBJECT_BACKTRACE_COUNT && nullptr != backtrace[bt_size]) {
key.bt_[bt_size] = backtrace[bt_size];
++bt_size;
}
key.bt_size_ = bt_size;
MEMCPY((char*)key.bt_, &object->data_[offset], AOBJECT_BACKTRACE_SIZE);
STRNCPY(key.label_, object->label_, sizeof(key.label_));
key.label_[sizeof(key.label_) - 1] = '\0';
ObMallocSampleValue *item = malloc_sample_map.get(key);
@ -581,6 +576,7 @@ void ObMemoryDump::handle(void *task)
ObLatchWGuard guard(iter_lock_, common::ObLatchIds::MEM_DUMP_ITER_LOCK);
std::swap(r_stat_, w_stat_);
}
ObFreeLogPrinter::get_instance().disable_free_log();
} else {
int fd = -1;
if (-1 == (fd = ::open(LOG_FILE,

118
deps/oblib/src/lib/alloc/ob_free_log_printer.cpp vendored Normal file
View File

@ -0,0 +1,118 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/alloc/ob_free_log_printer.h"
#include "lib/alloc/alloc_struct.h"
#include "lib/alloc/ob_malloc_sample_struct.h"
#include "lib/time/ob_time_utility.h"
#include "lib/utility/utility.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
ObFreeLogPrinter::ObFreeLogPrinter() : level_(DEFAULT), lock_()
{}
ObFreeLogPrinter& ObFreeLogPrinter::get_instance()
{
static ObFreeLogPrinter free_log_print;
return free_log_print;
}
int ObFreeLogPrinter::get_level()
{
int ret = DEFAULT;
int reason = g_alloc_failed_ctx().reason_;
switch (reason) {
case CTX_HOLD_REACH_LIMIT : {
ret = CTX;
break;
}
case TENANT_HOLD_REACH_LIMIT: {
ret = TENANT;
break;
}
case SERVER_HOLD_REACH_LIMIT: {
ret = SERVER;
break;
}
case PHYSICAL_MEMORY_EXHAUST: {
ret = SERVER;
break;
}
}
return ret;
}
void ObFreeLogPrinter::enable_free_log(int64_t tenant_id, int64_t ctx_id, int level)
{
bool has_lock = false;
if (DEFAULT == level_ && OB_SUCCESS == lock_.trylock()) {
DEFER(lock_.unlock());
last_enable_time_ = ObTimeUtility::current_time();
tenant_id_ = tenant_id;
ctx_id_ = ctx_id;
level_ = level;
has_lock = true;
}
if (has_lock) {
_OB_LOG(INFO, "start to print free log");
}
}
void ObFreeLogPrinter::disable_free_log()
{
bool has_lock = false;
if (DEFAULT != level_ && OB_SUCCESS == lock_.trylock()) {
DEFER(lock_.unlock());
level_ = DEFAULT;
has_lock = true;
}
if (has_lock) {
_OB_LOG(INFO, "finish to print free log");
}
}
void ObFreeLogPrinter::print_free_log(int64_t tenant_id, int64_t ctx_id, AObject *obj)
{
if (DEFAULT != level_ && obj->on_malloc_sample_) {
if (ObTimeUtility::current_time() - last_enable_time_ > MAX_FREE_LOG_TIME) {
disable_free_log();
} else {
bool allowed = false;
switch (level_) {
case CTX: {
allowed = tenant_id == tenant_id_ && ctx_id == ctx_id_;
break;
}
case TENANT: {
allowed = tenant_id == tenant_id_;
break;
}
case SERVER: {
allowed = true;
break;
}
}
if (allowed) {
char buf[MAX_BACKTRACE_LENGTH];
int64_t addrs[AOBJECT_BACKTRACE_COUNT];
int64_t offset = obj->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE;
MEMCPY((char*)addrs, &obj->data_[offset], sizeof(addrs));
IGNORE_RETURN parray(buf, sizeof(buf), addrs, ARRAYSIZEOF(addrs));
allow_next_syslog();
_OB_LOG(INFO, "[MEMORY][FREE_LOG] tenant_id=%ld, ctx_name=%s, mod=%s, alloc_bytes=%u, malloc_bt=%s\n",
tenant_id, get_global_ctx_info().get_ctx_name(ctx_id),
obj->label_, obj->alloc_bytes_, buf);
}
}
}
}

48
deps/oblib/src/lib/alloc/ob_free_log_printer.h vendored Normal file
View File

@ -0,0 +1,48 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_FREE_LOG_PRINTER_H_
#define _OB_FREE_LOG_PRINTER_H_
#include "lib/lock/ob_mutex.h"
namespace oceanbase
{
namespace lib
{
class AObject;
class ObFreeLogPrinter
{
public:
const int64_t MAX_FREE_LOG_TIME = 30 * 1000 * 1000; //30s
enum Level
{
DEFAULT = 0,
CTX = 1,
TENANT = 2,
SERVER = 3,
};
ObFreeLogPrinter();
static ObFreeLogPrinter& get_instance();
static int get_level();
void enable_free_log(int64_t tenant_id , int64_t ctx_id, int level);
void disable_free_log();
void print_free_log(int64_t tenant_id, int64_t ctx_id, AObject *obj);
private:
int64_t last_enable_time_;
int64_t tenant_id_;
int64_t ctx_id_;
int level_;
ObMutex lock_;
};
}
}
#endif

22
deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h vendored
View File

@ -22,6 +22,7 @@ namespace lib
{
static const int32_t AOBJECT_BACKTRACE_COUNT = 16;
static const int32_t AOBJECT_BACKTRACE_SIZE = sizeof(void*) * AOBJECT_BACKTRACE_COUNT;
static const int32_t MAX_BACKTRACE_LENGTH = 512;
static const int32_t MAX_MALLOC_SAMPLER_NUM = (1<<15) - 1;
class ObMallocSampleLimiter
@ -51,7 +52,6 @@ struct ObMallocSampleKey
bool operator==(const ObMallocSampleKey &other) const;
int64_t tenant_id_;
int64_t ctx_id_;
int32_t bt_size_;
char label_[lib::AOBJECT_LABEL_SIZE + 1];
void *bt_[AOBJECT_BACKTRACE_COUNT];
};
@ -140,7 +140,7 @@ inline int64_t ObMallocSampleKey::hash() const
hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val);
hash_val = murmurhash(&ctx_id_, sizeof(ctx_id_), hash_val);
hash_val = murmurhash(label_, sizeof(label_), hash_val);
hash_val = murmurhash(bt_, bt_size_ * sizeof(void*), hash_val);
hash_val = murmurhash(bt_, sizeof(bt_), hash_val);
return hash_val;
}
@ -148,28 +148,17 @@ inline bool ObMallocSampleKey::operator==(const ObMallocSampleKey &other) const
{
bool ret = true;
if (tenant_id_ != other.tenant_id_ || ctx_id_ != other.ctx_id_
|| 0 != STRNCMP(label_, other.label_, sizeof(label_))) {
|| 0 != STRNCMP(label_, other.label_, sizeof(label_))
|| 0 != MEMCMP((char*)bt_, (char*)other.bt_, sizeof(bt_))) {
ret = false;
}
if (ret) {
if (other.bt_size_ != bt_size_) {
ret = false;
} else {
for (int i = 0; i < bt_size_; ++i) {
if ((int64_t)bt_[i] != (int64_t)other.bt_[i]) {
ret = false;
break;
}
}
}
}
return ret;
}
#define ob_malloc_sample_backtrace(obj, size) \
{ \
if (OB_UNLIKELY(obj->on_malloc_sample_)) { \
void *addrs[100] = {nullptr}; \
void *addrs[100]; \
int bt_len = backtrace(addrs, ARRAYSIZEOF(addrs)); \
MEMCPY(&obj->data_[size], (char*)addrs, AOBJECT_BACKTRACE_SIZE); \
if (AOBJECT_BACKTRACE_COUNT > bt_len) { \
@ -177,7 +166,6 @@ inline bool ObMallocSampleKey::operator==(const ObMallocSampleKey &other) const
} \
} \
}
} // end of namespace lib
} // end of namespace oceanbase

13
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -14,6 +14,7 @@
#include "lib/alloc/ob_tenant_ctx_allocator.h"
#include "lib/alloc/ob_malloc_sample_struct.h"
#include "lib/alloc/ob_free_log_printer.h"
#include "lib/allocator/ob_mem_leak_checker.h"
#include "lib/allocator/ob_tc_malloc.h"
#include "lib/utility/ob_print_utils.h"
@ -418,6 +419,9 @@ void* ObTenantCtxAllocator::common_alloc(const int64_t size, const ObMemAttr &at
alloc_size - size + sizeof(AOBJECT_TAIL_MAGIC_CODE));
}
if (OB_UNLIKELY(nullptr == obj) && REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
int level = ObFreeLogPrinter::get_level();
ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_,
attr.ctx_id_, level);
const char *msg = alloc_failed_msg();
LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg));
_OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, "
@ -425,6 +429,7 @@ void* ObTenantCtxAllocator::common_alloc(const int64_t size, const ObMemAttr &at
attr.tenant_id_, attr.ctx_id_,
get_global_ctx_info().get_ctx_name(attr.ctx_id_),
ta.get_hold(), ta.get_limit(), ta.get_tenant_hold(), ta.get_tenant_limit());
ObMallocAllocator::get_instance()->print_tenant_memory_usage(attr.tenant_id_);
// 49 is the user defined signal to dump memory
raise(49);
}
@ -465,6 +470,9 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
SANITY_POISON((void*)upper_align((int64_t)obj->data_ + size, 8),
alloc_size - size + sizeof(AOBJECT_TAIL_MAGIC_CODE));
} else if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
int level = ObFreeLogPrinter::get_level();
ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_,
attr.ctx_id_, level);
const char *msg = alloc_failed_msg();
LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg));
_OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, "
@ -472,6 +480,7 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
attr.tenant_id_, attr.ctx_id_,
get_global_ctx_info().get_ctx_name(attr.ctx_id_),
ta.get_hold(), ta.get_limit(), ta.get_tenant_hold(), ta.get_tenant_limit());
ObMallocAllocator::get_instance()->print_tenant_memory_usage(attr.tenant_id_);
// 49 is the user defined signal to dump memory
raise(49);
}
@ -497,6 +506,10 @@ void ObTenantCtxAllocator::common_free(void *ptr)
abort_unless(block->obj_set_ != NULL);
ObjectSet *os = block->obj_set_;
auto blk_mgr = os->get_block_mgr();
int64_t tenant_id = blk_mgr->get_tenant_id();
int64_t ctx_id = blk_mgr->get_ctx_id();
ObFreeLogPrinter::get_instance().print_free_log(tenant_id, ctx_id, obj);
os->free_object(obj);
}
}

12
deps/oblib/src/lib/alloc/object_set.cpp vendored
View File

@ -56,7 +56,7 @@ AObject *ObjectSet::alloc_object(
const uint64_t adj_size = MAX(size, MIN_AOBJECT_SIZE);
const uint64_t all_size = align_up2(adj_size + AOBJECT_META_SIZE, 16);
const int64_t ctx_id = blk_mgr_->ctx_id_;
const int64_t ctx_id = blk_mgr_->get_ctx_id();
abort_unless(ctx_id == attr.ctx_id_);
if (OB_UNLIKELY(common::ObCtxIds::LIBEASY == ctx_id)) {
do_free_dirty_list();
@ -279,8 +279,8 @@ void ObjectSet::free_normal_object(AObject *obj)
normal_used_bytes_ -= obj->nobjs_ * AOBJECT_CELL_BYTES;
AObject *newobj = merge_obj(obj);
auto ctx_id = blk_mgr_->ctx_id_;
auto tenant_id = blk_mgr_->tenant_id_;
auto ctx_id = blk_mgr_->get_ctx_id();
auto tenant_id = blk_mgr_->get_tenant_id();
if (newobj->nobjs_ == cells_per_block_) {
hold_bytes_ -= ablock_size_;
normal_hold_bytes_ -= ablock_size_;
@ -376,7 +376,7 @@ void ObjectSet::free_object(AObject *obj)
memset(obj->data_, 0xAA, obj->alloc_bytes_);
}
#endif
const int64_t ctx_id = blk_mgr_->ctx_id_;
const int64_t ctx_id = blk_mgr_->get_ctx_id();
ObDisableDiagnoseGuard diagnose_disable_guard;
if (ctx_id == common::ObCtxIds::LIBEASY) {
if (locker_->trylock()) {
@ -560,8 +560,8 @@ bool ObjectSet::build_free_lists()
{
abort_unless(NULL == bm_ && NULL == free_lists_);
ObMemAttr attr;
attr.tenant_id_ = blk_mgr_->tenant_id_;
attr.ctx_id_ = blk_mgr_->ctx_id_;
attr.tenant_id_ = blk_mgr_->get_tenant_id();
attr.ctx_id_ = blk_mgr_->get_ctx_id();
attr.label_ = common::ObModIds::OB_OBJ_FREELISTS;
ABlock *new_block = alloc_block(sizeof (FreeList) * (cells_per_block_ + 1) +
sizeof (BitMap) + BitMap::buf_len(cells_per_block_ + 1), attr);

4
deps/oblib/src/lib/allocator/ob_allocator_v2.h vendored
View File

@ -147,8 +147,8 @@ inline int ObAllocator::init()
blk_mgr = pm;
pm_ = pm;
} else {
blk_mgr_.tenant_id_ = attr_.tenant_id_;
blk_mgr_.ctx_id_ = attr_.ctx_id_;
blk_mgr_.set_tenant_id(attr_.tenant_id_);
blk_mgr_.set_ctx_id(attr_.ctx_id_);
blk_mgr = &blk_mgr_;
}
if (OB_SUCC(ret)) {

6
deps/oblib/src/lib/allocator/ob_page_manager.h vendored
View File

@ -64,14 +64,12 @@ public:
return tenant_id_ < tenant_id ||
(tenant_id_ == tenant_id && id_ < id);
}
int set_tenant_ctx(const uint64_t tenant_id, const uint64_t ctx_id);
int set_tenant_ctx(const int64_t tenant_id, const int64_t ctx_id);
void set_max_chunk_cache_cnt(const int cnt)
{ bs_.set_max_chunk_cache_cnt(cnt); }
void reset();
int64_t get_hold() const;
int64_t get_tid() const { return tid_; }
int64_t get_tenant_id() const { return tenant_id_; }
int64_t get_ctx_id() const { return ctx_id_; }
// IBlockMgr interface
virtual ABlock *alloc_block(uint64_t size, const ObMemAttr &attr=default_memattr) override;
virtual void free_block(ABlock *block) override;
@ -135,7 +133,7 @@ inline ObPageManager::~ObPageManager()
}
}
inline int ObPageManager::set_tenant_ctx(const uint64_t tenant_id, const uint64_t ctx_id)
inline int ObPageManager::set_tenant_ctx(const int64_t tenant_id, const int64_t ctx_id)
{
int ret = OB_SUCCESS;
auto &pmc = ObPageManagerCenter::get_instance();

4
src/diagnose/lua/ob_lua_api.cpp
View File

@ -1964,8 +1964,8 @@ int select_malloc_sample_info(lua_State *L)
gen.next_column(it->first.label_);
// back_trace
{
char bt[512];
parray(bt, sizeof(bt), (int64_t*)*&(it->first.bt_), it->first.bt_size_);
char bt[MAX_BACKTRACE_LENGTH];
IGNORE_RETURN parray(bt, sizeof(bt), (int64_t*)it->first.bt_, AOBJECT_BACKTRACE_COUNT);
gen.next_column(bt);
}
// ctx_name

2
src/observer/virtual_table/ob_all_virtual_malloc_sample_info.cpp
View File

@ -118,7 +118,7 @@ int ObMallocSampleInfo::fill_row(ObNewRow *&row)
break;
}
case BACKTRACE: {
parray(bt_, sizeof(bt_), (int64_t*)*&(it_->first.bt_), it_->first.bt_size_);
IGNORE_RETURN parray(bt_, sizeof(bt_), (int64_t*)it_->first.bt_, AOBJECT_BACKTRACE_COUNT);
cells[i].set_varchar(bt_);
cells[i].set_collation_type(
ObCharset::get_default_collation(ObCharset::get_default_charset()));

2
src/observer/virtual_table/ob_all_virtual_malloc_sample_info.h
View File

@ -45,7 +45,7 @@ private:
ALLOC_BYTES,
};
char ip_buf_[common::OB_IP_STR_BUFF];
char bt_[512];
char bt_[lib::MAX_BACKTRACE_LENGTH];
lib::ObMallocSampleMap::const_iterator it_;
lib::ObMallocSampleMap malloc_sample_map_;
int64_t col_count_;