hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix the overflow check of AObject

Browse Source
This commit is contained in:
tushicheng
2024-04-12 10:16:15 +00:00
committed by ob-robot
parent f8ae0b11fe
commit fbfcd0feaa
10 changed files with 57 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deps/oblib/src/lib/CMakeLists.txt vendored
View File

@ -319,7 +319,6 @@ ob_set_subtarget(ob_malloc_object_list common_alloc
alloc/alloc_struct.cpp
alloc/block_set.cpp
alloc/memory_dump.cpp
alloc/ob_free_log_printer.cpp
alloc/ob_malloc_allocator.cpp
alloc/ob_malloc_callback.cpp
alloc/ob_malloc_sample_struct.cpp

12
deps/oblib/src/lib/alloc/alloc_struct.h vendored
View File

@ -150,13 +150,14 @@ struct ObMemAttr
prio_(prio),
use_500_(false),
expect_500_(true),
ignore_version_(ObMemVersionNode::tl_ignore_node)
ignore_version_(ObMemVersionNode::tl_ignore_node),
alloc_extra_info_(false)
{}
int64_t to_string(char* buf, const int64_t buf_len) const;
bool use_500() const { return use_500_; }
bool expect_500() const { return expect_500_; }
bool ignore_version() const { return ignore_version_; }
private:
public:
union {
char padding__[4];
struct {
@ -164,6 +165,7 @@ private:
uint8_t use_500_ : 1;
uint8_t expect_500_ : 1;
uint8_t ignore_version_ : 1;
uint8_t alloc_extra_info_ : 1;
};
};
};
@ -300,6 +302,7 @@ struct AObject {
OB_INLINE ABlock *block() const;
OB_INLINE uint64_t hold(uint32_t cells_per_block) const;
OB_INLINE ObLabel label() const;
OB_INLINE char *bt();
// members
union {
@ -357,6 +360,7 @@ static const uint32_t INTACT_MIDDLE_AOBJECT_SIZE = 64L << 10;
static const int32_t AOBJECT_BACKTRACE_COUNT = 16;
static const int32_t AOBJECT_BACKTRACE_SIZE = sizeof(void*) * AOBJECT_BACKTRACE_COUNT;
static const int32_t AOBJECT_EXTRA_INFO_SIZE = AOBJECT_BACKTRACE_SIZE;
static const int32_t MAX_BACKTRACE_LENGTH = 512;
@ -614,6 +618,10 @@ ObLabel AObject::label() const
{
return ObLabel(label_);
}
char *AObject::bt()
{
return &data_[alloc_bytes_ + AOBJECT_TAIL_SIZE];
}
class Label
{

14
deps/oblib/src/lib/alloc/memory_dump.cpp vendored
View File

@ -431,12 +431,11 @@ int label_stat(AChunk *chunk, ABlock *block, AObject *object,
LabelItem *litem = nullptr;
auto key = std::make_pair(*(uint64_t*)object->label_, *((uint64_t*)object->label_ + 1));
LabelInfoItem *linfoitem = lmap.get(key);
int64_t bt_size = object->on_malloc_sample_ ? AOBJECT_BACKTRACE_SIZE : 0;
if (NULL != linfoitem) {
// exist
litem = linfoitem->litem_;
litem->hold_ += hold;
litem->used_ += (object->alloc_bytes_ - bt_size);
litem->used_ += object->alloc_bytes_;
litem->count_++;
if (chunk != linfoitem->chunk_) {
litem->chunk_cnt_ += 1;
@ -456,7 +455,7 @@ int label_stat(AChunk *chunk, ABlock *block, AObject *object,
litem->str_[sizeof(litem->str_) - 1] = '\0';
litem->str_len_ = strlen(litem->str_);
litem->hold_ = hold;
litem->used_ = (object->alloc_bytes_ - bt_size);
litem->used_ = object->alloc_bytes_;
litem->count_ = 1;
litem->block_cnt_ = 1;
litem->chunk_cnt_ = 1;
@ -473,20 +472,19 @@ int malloc_sample_stat(uint64_t tenant_id, uint64_t ctx_id,
{
int ret = OB_SUCCESS;
if (object->in_use_ && object->on_malloc_sample_) {
int64_t offset = object->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE;
ObMallocSampleKey key;
key.tenant_id_ = tenant_id;
key.ctx_id_ = ctx_id;
MEMCPY((char*)key.bt_, &object->data_[offset], AOBJECT_BACKTRACE_SIZE);
MEMCPY((char*)key.bt_, object->bt(), AOBJECT_BACKTRACE_SIZE);
STRNCPY(key.label_, object->label_, sizeof(key.label_));
key.label_[sizeof(key.label_) - 1] = '\0';
ObMallocSampleValue *item = malloc_sample_map.get(key);
if (NULL != item) {
item->alloc_count_ += 1;
item->alloc_bytes_ += offset;
item->alloc_bytes_ += object->alloc_bytes_;
} else {
ObSignalHandlerGuard guard(ob_signal_handler);
ret = malloc_sample_map.set_refactored(key, ObMallocSampleValue(1, offset));
ret = malloc_sample_map.set_refactored(key, ObMallocSampleValue(1, object->alloc_bytes_));
}
}
return ret;
@ -572,7 +570,7 @@ void ObMemoryDump::handle(void *task)
has_memory_leak = true;
char bt[MAX_BACKTRACE_LENGTH] = {'\0'};
if (object->on_malloc_sample_) {
parray(bt, sizeof(bt), (int64_t*)&object->data_[object->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE], AOBJECT_BACKTRACE_COUNT);
parray(bt, sizeof(bt), (int64_t*)object->bt(), AOBJECT_BACKTRACE_COUNT);
}
allow_next_syslog();
LOG_WARN("SQL_MEMORY_LEAK", KP(object), K(tenant_id), K(ctx_id), K(object->version_), K(object->label_), K(bt));

118
deps/oblib/src/lib/alloc/ob_free_log_printer.cpp vendored
View File

@ -1,118 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/alloc/ob_free_log_printer.h"
#include "lib/alloc/alloc_struct.h"
#include "lib/alloc/ob_malloc_sample_struct.h"
#include "lib/time/ob_time_utility.h"
#include "lib/utility/utility.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
ObFreeLogPrinter::ObFreeLogPrinter() : level_(DEFAULT), lock_()
{}
ObFreeLogPrinter& ObFreeLogPrinter::get_instance()
{
static ObFreeLogPrinter free_log_print;
return free_log_print;
}
int ObFreeLogPrinter::get_level()
{
int ret = DEFAULT;
int reason = g_alloc_failed_ctx().reason_;
switch (reason) {
case CTX_HOLD_REACH_LIMIT : {
ret = CTX;
break;
}
case TENANT_HOLD_REACH_LIMIT: {
ret = TENANT;
break;
}
case SERVER_HOLD_REACH_LIMIT: {
ret = SERVER;
break;
}
case PHYSICAL_MEMORY_EXHAUST: {
ret = SERVER;
break;
}
}
return ret;
}
void ObFreeLogPrinter::enable_free_log(int64_t tenant_id, int64_t ctx_id, int level)
{
bool has_lock = false;
if (DEFAULT == level_ && OB_SUCCESS == lock_.trylock()) {
DEFER(lock_.unlock());
last_enable_time_ = ObTimeUtility::current_time();
tenant_id_ = tenant_id;
ctx_id_ = ctx_id;
level_ = level;
has_lock = true;
}
if (has_lock) {
_OB_LOG(INFO, "start to print free log");
}
}
void ObFreeLogPrinter::disable_free_log()
{
bool has_lock = false;
if (DEFAULT != level_ && OB_SUCCESS == lock_.trylock()) {
DEFER(lock_.unlock());
level_ = DEFAULT;
has_lock = true;
}
if (has_lock) {
_OB_LOG(INFO, "finish to print free log");
}
}
void ObFreeLogPrinter::print_free_log(int64_t tenant_id, int64_t ctx_id, AObject *obj)
{
if (DEFAULT != level_ && obj->on_malloc_sample_) {
if (ObTimeUtility::current_time() - last_enable_time_ > MAX_FREE_LOG_TIME) {
disable_free_log();
} else {
bool allowed = false;
switch (level_) {
case CTX: {
allowed = tenant_id == tenant_id_ && ctx_id == ctx_id_;
break;
}
case TENANT: {
allowed = tenant_id == tenant_id_;
break;
}
case SERVER: {
allowed = true;
break;
}
}
if (allowed) {
char buf[MAX_BACKTRACE_LENGTH];
int64_t addrs[AOBJECT_BACKTRACE_COUNT];
int64_t offset = obj->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE;
MEMCPY((char*)addrs, &obj->data_[offset], sizeof(addrs));
IGNORE_RETURN parray(buf, sizeof(buf), addrs, ARRAYSIZEOF(addrs));
allow_next_syslog();
_OB_LOG(INFO, "[MEM_FREE_LOG] tenant_id=%ld, ctx_name=%s, mod=%s, alloc_bytes=%u, malloc_bt=%s\n",
tenant_id, get_global_ctx_info().get_ctx_name(ctx_id),
obj->label_, obj->alloc_bytes_, buf);
}
}
}
}

48
deps/oblib/src/lib/alloc/ob_free_log_printer.h vendored
View File

@ -1,48 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_FREE_LOG_PRINTER_H_
#define _OB_FREE_LOG_PRINTER_H_
#include "lib/lock/ob_mutex.h"
namespace oceanbase
{
namespace lib
{
class AObject;
class ObFreeLogPrinter
{
public:
const int64_t MAX_FREE_LOG_TIME = 30 * 1000 * 1000; //30s
enum Level
{
DEFAULT = 0,
CTX = 1,
TENANT = 2,
SERVER = 3,
};
ObFreeLogPrinter();
static ObFreeLogPrinter& get_instance();
static int get_level();
void enable_free_log(int64_t tenant_id , int64_t ctx_id, int level);
void disable_free_log();
void print_free_log(int64_t tenant_id, int64_t ctx_id, AObject *obj);
private:
int64_t last_enable_time_;
int64_t tenant_id_;
int64_t ctx_id_;
int level_;
ObMutex lock_;
};
}
}
#endif

9
deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h vendored
View File

@ -159,15 +159,6 @@ inline bool ObMallocSampleKey::operator==(const ObMallocSampleKey &other) const
return ret;
}
#define ob_malloc_sample_backtrace(obj, size) \
{ \
if (OB_UNLIKELY(obj->on_malloc_sample_)) { \
void *addrs[100] = {nullptr}; \
int bt_len = ob_backtrace(addrs, ARRAYSIZEOF(addrs)); \
STATIC_ASSERT(AOBJECT_BACKTRACE_SIZE < sizeof(addrs), "AOBJECT_BACKTRACE_SIZE must be less than addrs!");\
MEMCPY(&obj->data_[size], (char*)addrs, AOBJECT_BACKTRACE_SIZE); \
} \
}
} // end of namespace lib
} // end of namespace oceanbase

39
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -413,7 +413,6 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
}
AObject *obj = NULL;
int64_t alloc_size = 0;
bool sample_allowed = false;
bool is_errsim = false;
if (NULL != ptr) {
@ -436,13 +435,14 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
#endif
ObLightBacktraceGuard light_backtrace_guard(is_memleak_light_backtrace_enabled()
&& ObCtxIds::GLIBC != attr.ctx_id_);
ObMemAttr inner_attr = attr;
if (OB_UNLIKELY(is_errsim)) {
} else {
BASIC_TIME_GUARD(time_guard, "ObMalloc");
DEFER(ObMallocTimeMonitor::get_instance().record_malloc_time(time_guard, size, attr));
sample_allowed = ObMallocSampleLimiter::malloc_sample_allowed(size, attr);
alloc_size = sample_allowed ? (size + AOBJECT_BACKTRACE_SIZE) : size;
obj = allocator.realloc_object(obj, alloc_size, attr);
DEFER(ObMallocTimeMonitor::get_instance().record_malloc_time(time_guard, size, inner_attr));
sample_allowed = ObMallocSampleLimiter::malloc_sample_allowed(size, inner_attr);
inner_attr.alloc_extra_info_ = sample_allowed;
obj = allocator.realloc_object(obj, size, inner_attr);
if(OB_ISNULL(obj)) {
int64_t total_size = 0;
if (g_alloc_failed_ctx().need_wash_block()) {
@ -453,31 +453,40 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
BASIC_TIME_GUARD_CLICK("WASH_CHUNK_END");
}
if (total_size > 0) {
obj = allocator.realloc_object(obj, alloc_size, attr);
obj = allocator.realloc_object(obj, size, inner_attr);
}
}
}
if (obj != NULL) {
obj->on_malloc_sample_ = sample_allowed;
ob_malloc_sample_backtrace(obj, size);
obj->ignore_version_ = attr.ignore_version() || ObMemVersionNode::tl_ignore_node;
if (inner_attr.label_.str_ != nullptr) {
STRNCPY(obj->label_, inner_attr.label_.str_, sizeof(obj->label_));
obj->label_[sizeof(obj->label_) - 1] = '\0';
} else {
MEMSET(obj->label_, '\0', sizeof(obj->label_));
}
if (sample_allowed) {
void *addrs[100] = {nullptr};
ob_backtrace(addrs, ARRAYSIZEOF(addrs));
STATIC_ASSERT(AOBJECT_BACKTRACE_SIZE < sizeof(addrs), "AOBJECT_BACKTRACE_SIZE must be less than addrs!");
MEMCPY(obj->bt(), (char*)addrs, AOBJECT_BACKTRACE_SIZE);
obj->on_malloc_sample_ = true;
}
obj->ignore_version_ = inner_attr.ignore_version() || ObMemVersionNode::tl_ignore_node;
if (!obj->ignore_version_) {
obj->version_ = ObMemVersionNode::tl_node->version_;
}
nptr = obj->data_;
get_mem_leak_checker().on_alloc(*obj, attr);
SANITY_POISON(obj, AOBJECT_HEADER_SIZE);
get_mem_leak_checker().on_alloc(*obj, inner_attr);
SANITY_POISON(obj, obj->nobjs_ * AOBJECT_CELL_BYTES);
SANITY_UNPOISON(obj->data_, size);
SANITY_POISON((void*)upper_align((int64_t)obj->data_ + size, 8),
alloc_size - size + sizeof(AOBJECT_TAIL_MAGIC_CODE));
} else if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
const char *msg = is_errsim ? "[ERRSIM] errsim inject memory error" : alloc_failed_msg();
LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg));
_OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, "
"ctx_limit=%ld, tenant_hold=%ld, tenant_limit=%ld",
attr.tenant_id_, attr.ctx_id_,
get_global_ctx_info().get_ctx_name(attr.ctx_id_),
inner_attr.tenant_id_, inner_attr.ctx_id_,
get_global_ctx_info().get_ctx_name(inner_attr.ctx_id_),
ta.get_hold(), ta.get_limit(), ta.get_tenant_hold(), ta.get_tenant_limit());
// 49 is the user defined signal to dump memory
raise(49);

19
deps/oblib/src/lib/alloc/object_mgr.cpp vendored
View File

@ -14,6 +14,7 @@
#include "lib/allocator/ob_ctx_define.h"
#include "lib/alloc/ob_malloc_allocator.h"
#include "lib/alloc/memory_sanity.h"
#include "lib/alloc/ob_tenant_ctx_allocator.h"
using namespace oceanbase;
using namespace lib;
@ -243,15 +244,13 @@ SubObjectMgr *ObjectMgr::create_sub_mgr()
attr.tenant_id_ = OB_SERVER_TENANT_ID;
attr.label_ = common::ObModIds::OB_TENANT_CTX_ALLOCATOR;
attr.ctx_id_ = ObCtxIds::DEFAULT_CTX_ID;
attr.ignore_version_ = true;
root_mgr.lock();
auto *obj = root_mgr.alloc_object(sizeof(SubObjectMgr), attr);
void *ptr = ObTenantCtxAllocator::common_realloc(NULL, sizeof(SubObjectMgr), attr, *(ta.ref_allocator()), root_mgr);
root_mgr.unlock();
if (OB_NOT_NULL(obj)) {
obj->ignore_version_ = true;
SANITY_UNPOISON(obj->data_, obj->alloc_bytes_);
sub_mgr = new (obj->data_) SubObjectMgr(ta_,
enable_no_log_,
ablock_size_, enable_dirty_list_, blk_mgr_);
if (OB_NOT_NULL(ptr)) {
sub_mgr = new (ptr) SubObjectMgr(ta_, enable_no_log_,
ablock_size_, enable_dirty_list_, blk_mgr_);
}
return sub_mgr;
}
@ -263,11 +262,7 @@ void ObjectMgr::destroy_sub_mgr(SubObjectMgr *sub_mgr)
ObCtxIds::DEFAULT_CTX_ID);
auto &root_mgr = static_cast<ObjectMgr&>(ta->get_block_mgr()).root_mgr_;
sub_mgr->~SubObjectMgr();
auto *obj = reinterpret_cast<AObject*>((char*)sub_mgr - AOBJECT_HEADER_SIZE);
abort_unless(obj->MAGIC_CODE_ == AOBJECT_MAGIC_CODE
|| obj->MAGIC_CODE_ == BIG_AOBJECT_MAGIC_CODE);
SANITY_POISON(obj->data_, obj->alloc_bytes_);
root_mgr.free_object(obj);
ObTenantCtxAllocator::common_free(sub_mgr);
}
}

4
deps/oblib/src/lib/alloc/object_mgr.h vendored
View File

@ -49,6 +49,10 @@ public:
{
return os_.alloc_object(size, attr);
}
OB_INLINE AObject *realloc_object(AObject *obj, const uint64_t size, const ObMemAttr &attr)
{
return os_.realloc_object(obj, size, attr);
}
void free_object(AObject *object);
OB_INLINE ABlock *alloc_block(uint64_t size, const ObMemAttr &attr) override
{

26
deps/oblib/src/lib/alloc/object_set.cpp vendored
View File

@ -55,7 +55,8 @@ AObject *ObjectSet::alloc_object(
const uint64_t size, const ObMemAttr &attr)
{
const uint64_t adj_size = MAX(size, MIN_AOBJECT_SIZE);
const uint64_t all_size = align_up2(adj_size + AOBJECT_META_SIZE, 16);
const uint64_t meta_size = AOBJECT_META_SIZE + (attr.alloc_extra_info_ ? AOBJECT_EXTRA_INFO_SIZE : 0);
const uint64_t all_size = align_up2(adj_size + meta_size, 16);
const int64_t ctx_id = blk_mgr_->get_ctx_id();
abort_unless(ctx_id == attr.ctx_id_);
@ -77,7 +78,7 @@ AObject *ObjectSet::alloc_object(
normal_used_bytes_ += obj->nobjs_ * AOBJECT_CELL_BYTES;
}
} else {
obj = alloc_big_object(adj_size, attr);
obj = alloc_big_object(all_size, attr);
abort_unless(NULL == obj || obj->in_use_);
}
@ -86,12 +87,6 @@ AObject *ObjectSet::alloc_object(
abort_unless(obj->is_valid());
reinterpret_cast<uint64_t&>(obj->data_[size]) = AOBJECT_TAIL_MAGIC_CODE;
obj->alloc_bytes_ = static_cast<uint32_t>(size);
if (attr.label_.str_ != nullptr) {
STRNCPY(&obj->label_[0], attr.label_.str_, sizeof(obj->label_));
obj->label_[sizeof(obj->label_) - 1] = '\0';
} else {
MEMSET(obj->label_, '\0', sizeof(obj->label_));
}
allocs_++;
alloc_bytes_ += size;
used_bytes_ += obj->hold(cells_per_block_);
@ -104,19 +99,12 @@ AObject *ObjectSet::realloc_object(
AObject *obj, const uint64_t size, const ObMemAttr &attr)
{
AObject *new_obj = NULL;
uint64_t copy_size = 0;
if (NULL == obj) {
new_obj = alloc_object(size, attr);
} else {
abort_unless(obj->is_valid());
if (obj->is_large_ != 0) {
copy_size = MIN(obj->alloc_bytes_, size);
} else {
copy_size = MIN(
size, (obj->nobjs_ - META_CELLS) * AOBJECT_CELL_BYTES);
}
uint64_t copy_size = MIN(obj->alloc_bytes_, size);
new_obj = alloc_object(size, attr);
if (NULL != new_obj && copy_size != 0) {
memmove(new_obj->data_, obj->data_, copy_size);
@ -336,13 +324,12 @@ void ObjectSet::free_block(ABlock *block)
AObject *ObjectSet::alloc_big_object(const uint64_t size, const ObMemAttr &attr)
{
AObject *obj = NULL;
ABlock *block = alloc_block(size + AOBJECT_META_SIZE, attr);
ABlock *block = alloc_block(size, attr);
if (NULL != block) {
obj = new (block->data()) AObject();
obj->is_large_ = true;
obj->in_use_ = true;
obj->alloc_bytes_ = static_cast<uint32_t>(size);
}
return obj;
@ -491,8 +478,7 @@ bool ObjectSet::check_has_unfree(char *first_label, char *first_bt)
}
if (obj->on_malloc_sample_ && '\0' == first_bt[0]) {
void *addrs[AOBJECT_BACKTRACE_COUNT];
int64_t offset = obj->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE;
MEMCPY((char*)addrs, &obj->data_[offset], AOBJECT_BACKTRACE_SIZE);
MEMCPY((char*)addrs, obj->bt(), AOBJECT_BACKTRACE_SIZE);
IGNORE_RETURN parray(first_bt, MAX_BACKTRACE_LENGTH, (int64_t*)addrs, AOBJECT_BACKTRACE_COUNT);
}
if (!has_unfree) {