hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] memleak with light_backtrace

Browse Source
This commit is contained in:
tushicheng 2024-03-28 07:15:48 +00:00 committed by ob-robot
parent db972bbeeb
commit 022fbefd85
21 changed files with 129 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -93,6 +93,8 @@ if (ENABLE_FATAL_ERROR_HANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DFATAL_ERROR_HANG")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_500_MEMORY_LIMIT")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_500_MEMORY_LIMIT")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_LIGHT_BACKTRACE")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_LIGHT_BACKTRACE")
endif()
if(OB_USE_ASAN)

4
deps/easy/CMakeLists.txt vendored
View File

@ -37,10 +37,10 @@ endif()
if (OB_USE_CLANG)
set(EASY_CC_WNO -fno-strict-aliasing -Wno-unused-variable -Wno-unused-function -fmax-type-align=8
-Wno-tautological-constant-out-of-range-compare)
-Wno-tautological-constant-out-of-range-compare -fno-omit-frame-pointer)
else()
set(EASY_CC_WNO -fno-strict-aliasing -Wno-unused-variable -Wno-implicit-function-declaration
-Wno-unused-but-set-variable -Wno-unused-function)
-Wno-unused-but-set-variable -Wno-unused-function -fno-omit-frame-pointer)
endif()

14
deps/oblib/src/lib/alloc/alloc_struct.cpp vendored
View File

@ -30,6 +30,8 @@ thread_local ObMemAttr ObMallocHookAttrGuard::tl_mem_attr(OB_SERVER_TENANT_ID,
static int64_t g_divisive_mem_size[OB_MAX_CPU_NUM];
static thread_local bool g_is_ob_mem_mgr_path = false;
static bool g_memleak_light_backtrace_enabled = false;
uint32_t ObMemVersionNode::global_version = 0;
__thread bool ObMemVersionNode::tl_ignore_node = true;
__thread ObMemVersionNode* ObMemVersionNode::tl_node = NULL;
@ -133,5 +135,17 @@ bool is_ob_mem_mgr_path()
return g_is_ob_mem_mgr_path;
}
void enable_memleak_light_backtrace(const bool enable)
{
#if defined(ENABLE_LIGHT_BACKTRACE)
g_memleak_light_backtrace_enabled = enable;
#else
UNUSED(enable);
#endif
}
bool is_memleak_light_backtrace_enabled()
{
return g_memleak_light_backtrace_enabled;
}
} // end of namespace lib
} // end of namespace oceanbase

35
deps/oblib/src/lib/alloc/alloc_struct.h vendored
View File

@ -355,6 +355,11 @@ static const uint32_t AOBJECT_META_SIZE = AOBJECT_HEADER_SIZE + AOBJECT_TAIL_SIZ
static const uint32_t INTACT_NORMAL_AOBJECT_SIZE = 8L << 10;
static const uint32_t INTACT_MIDDLE_AOBJECT_SIZE = 64L << 10;
static const int32_t AOBJECT_BACKTRACE_COUNT = 16;
static const int32_t AOBJECT_BACKTRACE_SIZE = sizeof(void*) * AOBJECT_BACKTRACE_COUNT;
static const int32_t MAX_BACKTRACE_LENGTH = 512;
static const uint32_t ABLOCK_HEADER_SIZE = sizeof(ABlock);
static const uint32_t ABLOCK_SIZE = INTACT_NORMAL_AOBJECT_SIZE;
@ -650,6 +655,33 @@ private:
ObMemAttr old_attr_;
};
class ObLightBacktraceGuard
{
public:
ObLightBacktraceGuard(const bool enable)
: last_(tl_enable())
{
tl_enable() = enable;
}
~ObLightBacktraceGuard()
{
tl_enable() = last_;
}
public:
static bool is_enabled()
{
return tl_enable();
}
private:
static bool &tl_enable()
{
static __thread bool enable = false;
return enable;
}
private:
const bool last_;
};
extern void inc_divisive_mem_size(const int64_t size);
extern void dec_divisive_mem_size(const int64_t size);
extern int64_t get_divisive_mem_size();
@ -658,6 +690,9 @@ extern void set_ob_mem_mgr_path();
extern void unset_ob_mem_mgr_path();
extern bool is_ob_mem_mgr_path();
extern void enable_memleak_light_backtrace(const bool);
extern bool is_memleak_light_backtrace_enabled();
#define FORCE_EXPLICT_500_MALLOC() \
OB_UNLIKELY(oceanbase::lib::ObMallocAllocator::get_instance()->force_explict_500_malloc_)

9
deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp vendored
View File

@ -715,18 +715,21 @@ int ObMallocAllocator::recycle_tenant_allocator(uint64_t tenant_id)
if (NULL == ctx_allocator) {
ctx_allocator = &ta[ctx_id];
char first_label[AOBJECT_LABEL_SIZE + 1] = {'\0'};
bool has_unfree = ctx_allocator->check_has_unfree(first_label);
char first_bt[MAX_BACKTRACE_LENGTH] = {'\0'};
bool has_unfree = ctx_allocator->check_has_unfree(first_label, first_bt);
if (has_unfree) {
if (ObCtxIds::GLIBC == ctx_id
&& 0 == strncmp("Pl", first_label, 2)
&& pl_leaked_times_++ < 10) {
LOG_WARN("tenant memory leak!!!", K(tenant_id), K(ctx_id),
"ctx_name", get_global_ctx_info().get_ctx_name(ctx_id),
"label", first_label);
"label", first_label,
"backtrace", first_bt);
} else {
LOG_ERROR("tenant memory leak!!!", K(tenant_id), K(ctx_id),
"ctx_name", get_global_ctx_info().get_ctx_name(ctx_id),
"label", first_label);
"label", first_label,
"backtrace", first_bt);
}
tas[ctx_id] = ctx_allocator;
}

12
deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h vendored
View File

@ -20,9 +20,6 @@ namespace oceanbase
{
namespace lib
{
static const int32_t AOBJECT_BACKTRACE_COUNT = 16;
static const int32_t AOBJECT_BACKTRACE_SIZE = sizeof(void*) * AOBJECT_BACKTRACE_COUNT;
static const int32_t MAX_BACKTRACE_LENGTH = 512;
static const int32_t MAX_MALLOC_SAMPLER_NUM = (1<<15) - 1;
class ObMallocSampleLimiter
@ -100,7 +97,10 @@ inline bool ObMallocSampleLimiter::try_acquire(int64_t alloc_bytes)
inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, const ObMemAttr &attr)
{
bool ret = false;
if (OB_UNLIKELY(INTERVAL_UPPER_LIMIT == min_malloc_sample_interval)) {
if (ObLightBacktraceGuard::is_enabled()) {
// light_backtrace can sample all.
ret = true;
} else if (OB_UNLIKELY(INTERVAL_UPPER_LIMIT == min_malloc_sample_interval)) {
// Zero sample mode.
} else if (OB_UNLIKELY(MUST_SAMPLE_SIZE <= size)) {
// Full sample when size is bigger than 16M.
@ -163,8 +163,8 @@ inline bool ObMallocSampleKey::operator==(const ObMallocSampleKey &other) const
{ \
if (OB_UNLIKELY(obj->on_malloc_sample_)) { \
void *addrs[100] = {nullptr}; \
int bt_len = OB_BACKTRACE_M(addrs, ARRAYSIZEOF(addrs)); \
STATIC_ASSERT(AOBJECT_BACKTRACE_SIZE < sizeof(addrs), "AOBJECT_BACKTRACE_SIZE must be less than addrs!"); \
int bt_len = ob_backtrace(addrs, ARRAYSIZEOF(addrs)); \
STATIC_ASSERT(AOBJECT_BACKTRACE_SIZE < sizeof(addrs), "AOBJECT_BACKTRACE_SIZE must be less than addrs!");\
MEMCPY(&obj->data_[size], (char*)addrs, AOBJECT_BACKTRACE_SIZE); \
} \
}

3
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -434,7 +434,8 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
is_errsim = true;
}
#endif
ObLightBacktraceGuard light_backtrace_guard(is_memleak_light_backtrace_enabled()
&& ObCtxIds::GLIBC != attr.ctx_id_);
if (OB_UNLIKELY(is_errsim)) {
} else {
BASIC_TIME_GUARD(time_guard, "ObMalloc");

6
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.h vendored
View File

@ -268,13 +268,13 @@ public:
int iter_label(VisitFunc func) const;
int64_t sync_wash(int64_t wash_size);
int64_t sync_wash();
bool check_has_unfree(char *first_label)
bool check_has_unfree(char *first_label, char *first_bt)
{
bool has_unfree = obj_mgr_.check_has_unfree();
if (has_unfree) {
bool tmp_has_unfree = obj_mgr_.check_has_unfree(first_label);
bool tmp_has_unfree = obj_mgr_.check_has_unfree(first_label, first_bt);
for (int i = 0; i < ObSubCtxIds::MAX_SUB_CTX_ID && !tmp_has_unfree; ++i) {
tmp_has_unfree = obj_mgrs_[i].check_has_unfree(first_label);
tmp_has_unfree = obj_mgrs_[i].check_has_unfree(first_label, first_bt);
}
}
return has_unfree;

4
deps/oblib/src/lib/alloc/object_mgr.cpp vendored
View File

@ -332,7 +332,7 @@ bool ObjectMgr::check_has_unfree()
return has_unfree;
}
bool ObjectMgr::check_has_unfree(char *first_label)
bool ObjectMgr::check_has_unfree(char *first_label, char *first_bt)
{
bool has_unfree = false;
for (uint64_t idx = 0; idx < ATOMIC_LOAD(&sub_cnt_) && !has_unfree; idx++) {
@ -342,7 +342,7 @@ bool ObjectMgr::check_has_unfree(char *first_label)
} else {
sub_mgr->lock();
DEFER(sub_mgr->unlock());
has_unfree = sub_mgr->check_has_unfree(first_label);
has_unfree = sub_mgr->check_has_unfree(first_label, first_bt);
}
}
return has_unfree;

6
deps/oblib/src/lib/alloc/object_mgr.h vendored
View File

@ -63,9 +63,9 @@ public:
{
return bs_.check_has_unfree();
}
OB_INLINE bool check_has_unfree(char *first_label)
OB_INLINE bool check_has_unfree(char *first_label, char *first_bt)
{
return os_.check_has_unfree(first_label);
return os_.check_has_unfree(first_label, first_bt);
}
private:
ObTenantCtxAllocator &ta_;
@ -116,7 +116,7 @@ public:
int64_t sync_wash(int64_t wash_size) override;
Stat get_stat();
bool check_has_unfree();
bool check_has_unfree(char *first_label);
bool check_has_unfree(char *first_label, char *first_bt);
private:
SubObjectMgr *create_sub_mgr();
void destroy_sub_mgr(SubObjectMgr *sub_mgr);

15
deps/oblib/src/lib/alloc/object_set.cpp vendored
View File

@ -465,7 +465,7 @@ void ObjectSet::do_free_dirty_list()
}
}
bool ObjectSet::check_has_unfree(char *first_label)
bool ObjectSet::check_has_unfree(char *first_label, char *first_bt)
{
SANITY_DISABLE_CHECK_RANGE(); // prevent sanity_check_range
bool has_unfree = false;
@ -489,6 +489,12 @@ bool ObjectSet::check_has_unfree(char *first_label)
if ('\0' == first_label[0]) {
STRCPY(first_label, obj->label_);
}
if (obj->on_malloc_sample_ && '\0' == first_bt[0]) {
void *addrs[AOBJECT_BACKTRACE_COUNT];
int64_t offset = obj->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE;
MEMCPY((char*)addrs, &obj->data_[offset], AOBJECT_BACKTRACE_SIZE);
IGNORE_RETURN parray(first_bt, MAX_BACKTRACE_LENGTH, (int64_t*)addrs, AOBJECT_BACKTRACE_COUNT);
}
if (!has_unfree) {
has_unfree = true;
}
@ -517,16 +523,17 @@ void ObjectSet::reset()
const static int buf_len = 256;
char buf[buf_len] = {'\0'};
char first_label[AOBJECT_LABEL_SIZE + 1] = {'\0'};
bool has_unfree = check_has_unfree(first_label);
char first_bt[MAX_BACKTRACE_LENGTH] = {'\0'};
bool has_unfree = check_has_unfree(first_label, first_bt);
if (has_unfree) {
if (context_check) {
const StaticInfo &static_info = mem_context_->get_static_info();
const DynamicInfo &dynamic_info = mem_context_->get_dynamic_info();
int64_t pos = snprintf(buf, buf_len,
"context: %p, label: %s, static_id: 0x%lx, "
"context: %p, label: %s, backtrace: %s, static_id: 0x%lx, "
"static_info:{filename: %s, line: %d, function: %s}, "
"dynamic_info:{tid: %ld, cid: %ld, create_time: %ld}",
mem_context_, first_label,
mem_context_, first_label, first_bt,
mem_context_->get_static_id(),
static_info.filename_, static_info.line_, static_info.function_,
dynamic_info.tid_, dynamic_info.cid_, dynamic_info.create_time_);

2
deps/oblib/src/lib/alloc/object_set.h vendored
View File

@ -70,7 +70,7 @@ public:
inline int64_t get_normal_hold() const;
inline int64_t get_normal_used() const;
inline int64_t get_normal_alloc() const;
bool check_has_unfree(char *first_label);
bool check_has_unfree(char *first_label, char *first_bt);
private:
AObject *alloc_normal_object(const uint32_t cls, const ObMemAttr &attr);

31
deps/oblib/src/lib/utility/ob_backtrace.cpp vendored
View File

@ -21,17 +21,34 @@
#include "lib/utility/ob_defer.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/coro/co_var.h"
#include "common/ob_common_utility.h"
namespace oceanbase
{
namespace common
{
int ob_backtrace(void **buffer, int size)
int light_backtrace(void **buffer, int size)
{
int rv = 0;
if (OB_LIKELY(g_enable_backtrace)) {
rv = backtrace(buffer, size);
if (rv < size) {
buffer[rv++] = (void*)light_backtrace;
}
void *stack_addr = nullptr;
size_t stack_size = 0;
if (OB_LIKELY(OB_SUCCESS == get_stackattr(stack_addr, stack_size))) {
#define addr_in_stack(addr) (addr >= (int64_t)stack_addr && addr < (int64_t)stack_addr + stack_size)
int64_t rbp = (int64_t)__builtin_frame_address(0);
while (rbp != 0 && rv < size) {
if (!addr_in_stack(*(int64_t*)rbp) &&
!FALSE_IT(rbp += 16) &&
!addr_in_stack(*(int64_t*)rbp)) {
break;
} else {
int64_t return_addr = rbp + 8;
buffer[rv++] = (void*)*(int64_t*)return_addr;
rbp = *(int64_t*)rbp;
}
}
}
return rv;
}
@ -127,13 +144,13 @@ RLOCAL(ByteBuf<LBT_BUFFER_LENGTH>, buffer);
char *lbt()
{
int size = OB_BACKTRACE_M(addrs, MAX_ADDRS_COUNT);
int size = ob_backtrace(addrs, MAX_ADDRS_COUNT);
return parray(*&buffer, LBT_BUFFER_LENGTH, (int64_t *)addrs, size);
}
char *lbt(char *buf, int32_t len)
{
int size = OB_BACKTRACE_M(addrs, MAX_ADDRS_COUNT);
int size = ob_backtrace(addrs, MAX_ADDRS_COUNT);
return parray(buf, len, (int64_t *)addrs, size);
}
@ -178,7 +195,7 @@ void addrs_to_offsets(void **buffer, int size)
EXTERN_C_BEGIN
int ob_backtrace_c(void **buffer, int size)
{
return OB_BACKTRACE_M(buffer, size);
return ob_backtrace(buffer, size);
}
char *parray_c(char *buf, int64_t len, int64_t *array, int size)
{

20
deps/oblib/src/lib/utility/ob_backtrace.h vendored
View File

@ -21,15 +21,19 @@ namespace common
void init_proc_map_info();
extern bool g_enable_backtrace;
const int64_t LBT_BUFFER_LENGTH = 1024;
int ob_backtrace(void **buffer, int size);
int light_backtrace(void **buffer, int size);
// save one layer of call stack
#define OB_BACKTRACE_M(buffer, size) \
({ \
int rv = 0; \
if (OB_LIKELY(::oceanbase::common::g_enable_backtrace)) { \
rv = backtrace(buffer, size); \
} \
rv; \
#define ob_backtrace(buffer, size) \
({ \
int rv = 0; \
if (OB_LIKELY(::oceanbase::common::g_enable_backtrace)) { \
if (::oceanbase::lib::ObLightBacktraceGuard::is_enabled()) {\
rv = light_backtrace(buffer, size); \
} else { \
rv = backtrace(buffer, size); \
} \
} \
rv; \
})
char *lbt();
char *lbt(char *buf, int32_t len);

21
deps/oblib/src/lib/utility/ob_preload.h vendored
View File

@ -20,38 +20,17 @@
#include <stdarg.h>
#include <execinfo.h>
inline int64_t &bt(const char *msg)
{
int i = 0;
static int64_t enable_bt = 0;
if (enable_bt > 0) {
void *buffer[100];
int size = ob_backtrace(buffer, 100);
char **strings = backtrace_symbols(buffer, size);
_OB_LOG(DEBUG, "%s", msg);
if (NULL != strings) {
for (i = 0; i < size; i++) {
_OB_LOG(DEBUG, "BT[%d] @[%s]", i, strings[i]);
}
free(strings);
}
}
return enable_bt;
}
#ifdef __ENABLE_PRELOAD__
inline int pthread_key_create(pthread_key_t *key, void (*destructor)(void *))
{
int (*real_func)(pthread_key_t *key,
void (*destructor)(void *)) = (typeof(real_func))dlsym(RTLD_NEXT, "pthread_key_create");
bt("pthread_key_create");
return real_func(key, destructor);
}
inline int pthread_key_delete(pthread_key_t key)
{
int (*real_func)(pthread_key_t key) = (typeof(real_func))dlsym(RTLD_NEXT, "pthread_key_delete");
bt("pthread_key_delete");
return real_func(key);
}
#endif /* __ENABLE_PRELOAD__ */

1
src/observer/ob_server.cpp
View File

@ -2083,6 +2083,7 @@ int ObServer::init_pre_setting()
reset_mem_leak_checker_label(GCONF.leak_mod_to_check.str());
ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval,
GCONF._min_malloc_sample_interval);
enable_memleak_light_backtrace(GCONF._enable_memleak_light_backtrace);
// oblog configuration
if (OB_SUCC(ret)) {

1
src/observer/ob_server_reload_config.cpp
View File

@ -151,6 +151,7 @@ int ObServerReloadConfig::operator()()
#endif
ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval,
GCONF._min_malloc_sample_interval);
enable_memleak_light_backtrace(GCONF._enable_memleak_light_backtrace);
if (!is_arbitration_mode) {
ObIOConfig io_config;
int64_t cpu_cnt = GCONF.cpu_count;

8
src/rootserver/ob_root_service.cpp
View File

@ -6026,14 +6026,6 @@ int ObRootService::fetch_sys_tenant_ls_info()
return ret;
}
int ObRootService::not_implement()
{
int ret = OB_NOT_IMPLEMENT;
bt("not implement");
LOG_WARN("rpc not implemented", K(ret));
return ret;
}
ObRootService::ObRestartTask::ObRestartTask(ObRootService &root_service)
:ObAsyncTimerTask(root_service.task_queue_),
root_service_(root_service)

3
src/rootserver/ob_root_service.h
View File

@ -423,9 +423,6 @@ public:
const ObIArray<ObAddr> &servers,
const ObZone &zone);
// not implemented rpc, helper function for rs rpc processor define.
int not_implement();
int execute_bootstrap(const obrpc::ObBootstrapArg &arg);
#ifdef OB_BUILD_TDE_SECURITY
int check_sys_tenant_initial_master_key_valid();

3
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1905,3 +1905,6 @@ DEF_BOOL(strict_check_os_params, OB_CLUSTER_PARAMETER, "False",
"A switch that determines whether to enable strict OS parameter check mode, defaulting to true and can be set to false to bypass strict checks."
"Value: True: allowed; False: allowed but not suggested",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::STATIC_EFFECTIVE));
DEF_BOOL(_enable_memleak_light_backtrace, OB_CLUSTER_PARAMETER, "True",
"specifies whether allow memleak to get the backtrace of malloc by light_backtrace",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -291,6 +291,7 @@ _enable_easy_keepalive
_enable_hash_join_hasher
_enable_hash_join_processor
_enable_in_range_optimization
_enable_memleak_light_backtrace
_enable_newsort
_enable_new_sql_nio
_enable_optimizer_qualify_filter