hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add sys_parameter: _force_explict_500_malloc && Make the 500 anti-fallback test effective

Browse Source
This commit is contained in:
obdev
2023-06-06 04:12:22 +00:00
committed by ob-robot
parent 56e4920428
commit 6307df5985
18 changed files with 95 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CMakeLists.txt
View File

@ -42,11 +42,6 @@ if(ENABLE_LATCH_DIAGNOSE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_LATCH_DIAGNOSE")
endif()
if(ENABLE_500_FALLBACK)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_500_FALLBACK")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_500_FALLBACK")
endif()
if(ENABLE_SMART_VAR_CHECK)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_SMART_VAR_CHECK")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SMART_VAR_CHECK")

5
deps/oblib/src/lib/alloc/alloc_struct.h vendored
View File

@ -142,9 +142,7 @@ private:
inline ObMemAttr DoNotUseMe(ObMemAttr &attr)
{
#ifdef ENABLE_500_FALLBACK
attr.use_500_ = true;
#endif
return attr;
}
@ -609,6 +607,9 @@ private:
ObMemAttr old_attr_;
};
#define FORCE_EXPLICT_500_MALLOC() \
OB_UNLIKELY(oceanbase::lib::ObMallocAllocator::get_instance()->force_explict_500_malloc_)
} // end of namespace lib
} // end of namespace oceanbase

101
deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp vendored
View File

@ -16,6 +16,7 @@
#include "lib/alloc/alloc_struct.h"
#include "lib/alloc/object_set.h"
#include "lib/alloc/memory_sanity.h"
#include "lib/alloc/memory_dump.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/allocator/ob_mem_leak_checker.h"
#include "lib/allocator/ob_page_manager.h"
@ -90,7 +91,7 @@ void *ObMallocAllocator::alloc(const int64_t size)
return alloc(size, attr);
}
void *ObMallocAllocator::alloc(const int64_t size, const oceanbase::lib::ObMemAttr &attr)
void *ObMallocAllocator::alloc(const int64_t size, const oceanbase::lib::ObMemAttr &_attr)
{
#ifdef OB_USE_ASAN
UNUSED(attr);
@ -101,20 +102,20 @@ void *ObMallocAllocator::alloc(const int64_t size, const oceanbase::lib::ObMemAt
int ret = OB_E(EventTable::EN_4) OB_SUCCESS;
void *ptr = NULL;
ObTenantCtxAllocatorGuard allocator = NULL;
oceanbase::lib::ObMemAttr inner_attr = attr;
#ifdef ENABLE_500_FALLBACK
const bool do_not_use_me = OB_SERVER_TENANT_ID == attr.tenant_id_ && !attr.use_500() &&
mtl_id() != OB_SERVER_TENANT_ID;
if (do_not_use_me) {
inner_attr.tenant_id_ = mtl_id();
inner_attr.ctx_id_ = ObCtxIds::DO_NOT_USE_ME;
lib::ObMemAttr attr = _attr;
if (OB_INVALID_TENANT_ID == attr.tenant_id_) {
LOG_ERROR("invalid tenant id", K(attr.tenant_id_));
attr.tenant_id_ = OB_SERVER_TENANT_ID;
}
#else
const bool do_not_use_me = false;
#endif
if (OB_INVALID_TENANT_ID == inner_attr.tenant_id_) {
inner_attr.tenant_id_ = OB_SERVER_TENANT_ID;
LOG_ERROR("invalid tenant id", K(attr.tenant_id_), K(inner_attr.tenant_id_), K(ret));
lib::ObMemAttr inner_attr = attr;
bool do_not_use_me = false;
if (FORCE_EXPLICT_500_MALLOC()) {
do_not_use_me = OB_SERVER_TENANT_ID == attr.tenant_id_ && !attr.use_500() &&
ob_thread_tenant_id() != OB_SERVER_TENANT_ID;
if (do_not_use_me) {
inner_attr.tenant_id_ = ob_thread_tenant_id();
inner_attr.ctx_id_ = ObCtxIds::DO_NOT_USE_ME;
}
}
if (OB_SUCCESS != ret) {
} else if (OB_UNLIKELY(0 == inner_attr.tenant_id_)
@ -138,13 +139,15 @@ void *ObMallocAllocator::alloc(const int64_t size, const oceanbase::lib::ObMemAt
if (OB_SUCC(ret)) {
if (do_not_use_me) {
if (OB_ISNULL(ptr = allocator->alloc(size, inner_attr))) {
inner_attr = attr;
allocator = get_tenant_ctx_allocator(inner_attr.tenant_id_, inner_attr.ctx_id_);
}
ptr = allocator->alloc(size, inner_attr);
}
if (!ptr) {
ptr = allocator->alloc(size, inner_attr);
inner_attr = attr;
allocator = get_tenant_ctx_allocator(inner_attr.tenant_id_, inner_attr.ctx_id_);
if (OB_ISNULL(allocator)) {
} else {
ptr = allocator->alloc(size, inner_attr);
}
}
}
@ -316,10 +319,8 @@ int ObMallocAllocator::create_tenant_allocator(uint64_t tenant_id, void *buf,
ObTenantCtxAllocator(tenant_id, ctx_id);
if (OB_FAIL(ctx_allocator->set_tenant_memory_mgr())) {
LOG_ERROR("set_tenant_memory_mgr failed", K(ret));
#ifdef ENABLE_500_FALLBACK
} else if (ObCtxIds::DO_NOT_USE_ME == ctx_id) {
ctx_allocator->set_limit(50L<<20);
#endif
ctx_allocator->set_limit(256L<<20);
}
}
if (OB_SUCC(ret)) {
@ -606,11 +607,6 @@ void ObMallocAllocator::add_tenant_allocator_unrecycled(ObTenantCtxAllocator *al
if (enable_tenant_leak_memory_protection_) {
modify_tenant_memory_access_permission(allocator, false);
}
#endif
#ifdef ENABLE_500_FALLBACK
for (int64_t ctx_id = 0; ctx_id < ObCtxIds::MAX_CTX_ID; ctx_id++) {
allocator[ctx_id].set_deleted(true);
}
#endif
ObDisableDiagnoseGuard disable_diagnose_guard;
ObLatchWGuard guard(unrecycled_lock_, ObLatchIds::OB_ALLOCATOR_LOCK);
@ -637,11 +633,6 @@ ObTenantCtxAllocator *ObMallocAllocator::take_off_tenant_allocator_unrecycled(ui
}
}
if (ta != NULL) {
#ifdef ENABLE_500_FALLBACK
for (int64_t ctx_id = 0; ctx_id < ObCtxIds::MAX_CTX_ID; ctx_id++) {
ta[ctx_id].set_deleted(false);
}
#endif
#ifdef ENABLE_SANITY
modify_tenant_memory_access_permission(ta, true);
#endif
@ -779,51 +770,5 @@ void ObMallocAllocator::modify_tenant_memory_access_permission(ObTenantCtxAlloca
}
#endif
#ifdef ENABLE_500_FALLBACK
int64_t __attribute__((weak)) mtl_id()
{
return OB_SERVER_TENANT_ID;
}
#endif
void ObMallocAllocator::print_malloc_sample(uint64_t tenant_id) const
{
UNUSED(tenant_id);
#ifdef ENABLE_500_FALLBACK
int ret = OB_SUCCESS;
auto &mem_dump = ObMemoryDump().get_instance();
if (OB_UNLIKELY(!mem_dump.is_inited())) {
ret = OB_NOT_INIT;
SERVER_LOG(WARN, "mem dump not inited", K(ret));
} else {
ObLatchRGuard guard(mem_dump.iter_lock_, common::ObLatchIds::MEM_DUMP_ITER_LOCK);
auto &sample_map = mem_dump.r_stat_->malloc_sample_map_;
char buf[4096];
int64_t buf_len = sizeof(buf);
int64_t pos = 0;
for (auto it = sample_map.begin(); OB_SUCC(ret) && it != sample_map.end(); it++) {
if (it->first.tenant_id_ == tenant_id) {
void **bt[ARRAYSIZEOF(it->first.bt_)];
MEMCPY(bt, it->first.bt_, sizeof(bt));
char bt_str[512];
parray(bt_str, sizeof(bt_str), (int64_t*)bt, ARRAYSIZEOF(bt));
ret = databuff_printf(buf, buf_len, pos,
"[MEMORY][LEAK_INFO] tenant=%'10ld ctx=%20s mod=%15s alloc_bytes=%'10ld lbt=%s\n",
it->first.tenant_id_,
get_global_ctx_info().get_ctx_name(it->first.ctx_id_),
it->first.label_, it->second.alloc_bytes_, bt_str);
if (OB_SUCC(ret) && pos > sizeof(buf)/2) {
_LOG_INFO("\n%.*s", static_cast<int32_t>(pos), buf);
pos = 0;
}
}
}
if (OB_SUCC(ret) && pos > 0) {
_LOG_INFO("\n%.*s", static_cast<int32_t>(pos), buf);
}
}
#endif
}
} // end of namespace lib
} // end of namespace oceanbase

3
deps/oblib/src/lib/alloc/ob_malloc_allocator.h vendored
View File

@ -131,7 +131,6 @@ public:
void print_tenant_ctx_memory_usage(uint64_t tenant_id) const;
void print_tenant_memory_usage(uint64_t tenant_id) const;
void print_malloc_sample(uint64_t tenant_id) const;
int set_tenant_ctx_idle(
const uint64_t tenant_id, const uint64_t ctx_id, const int64_t size, const bool reserve = false);
int64_t sync_wash(uint64_t tenant_id, uint64_t from_ctx_id, int64_t wash_size);
@ -156,6 +155,8 @@ private:
public:
bool enable_tenant_leak_memory_protection_ = true;
#endif
public:
bool force_explict_500_malloc_ = false;
private:
DISALLOW_COPY_AND_ASSIGN(ObMallocAllocator);
class BucketLock

10
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -511,14 +511,6 @@ void ObTenantCtxAllocator::common_free(void *ptr)
abort_unless(block->obj_set_ != NULL);
ObjectSet *os = block->obj_set_;
#ifdef ENABLE_500_FALLBACK
auto *ta = chunk->block_set_->get_tenant_ctx_allocator();
const bool do_free = OB_LIKELY(!ta->has_deleted()) || ta->get_tenant_id() <= OB_USER_TENANT_ID;
#else
const bool do_free = true;
#endif
if (do_free) {
os->free_object(obj);
}
os->free_object(obj);
}
}

2
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.h vendored
View File

@ -81,8 +81,6 @@ public:
{
return reinterpret_cast<ObTenantCtxAllocator*&>(next_);
}
void set_deleted(bool deleted) { deleted_ = deleted; }
bool has_deleted() const { return deleted_; }
// will delete it
virtual void *alloc(const int64_t size)

26
deps/oblib/src/lib/allocator/ob_allocator_v2.cpp vendored
View File

@ -30,12 +30,28 @@ void *ObAllocator::alloc(const int64_t size, const ObMemAttr &attr)
ret = init();
}
if (OB_SUCC(ret)) {
ObMemAttr inner_attr = attr_;
if (attr.label_.is_valid()) {
inner_attr.label_ = attr.label_;
if (do_not_use_me_) {
ObMemAttr inner_attr = nattr_;
if (attr.label_.is_valid()) {
inner_attr.label_ = attr.label_;
}
auto ta = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(inner_attr.tenant_id_,
inner_attr.ctx_id_);
if (ta != NULL) {
ptr = ObTenantCtxAllocator::common_alloc(size, inner_attr, *(ta.ref_allocator()), nos_);
}
}
if (OB_LIKELY(!ptr)) {
ObMemAttr inner_attr = attr_;
if (attr.label_.is_valid()) {
inner_attr.label_ = attr.label_;
}
auto ta = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(inner_attr.tenant_id_,
inner_attr.ctx_id_);
if (ta != NULL) {
ptr = ObTenantCtxAllocator::common_alloc(size, inner_attr, *(ta.ref_allocator()), os_);
}
}
auto ta = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(attr_.tenant_id_, attr_.ctx_id_);
ptr = ObTenantCtxAllocator::common_alloc(size, inner_attr, *(ta.ref_allocator()), os_);
}
return ptr;
}

25
deps/oblib/src/lib/allocator/ob_allocator_v2.h vendored
View File

@ -67,7 +67,9 @@ private:
void reset() override { os_.reset(); }
private:
__MemoryContext__ *mem_context_;
bool do_not_use_me_;
ObMemAttr attr_;
ObMemAttr nattr_;
const bool use_pm_;
void *pm_;
lib::ISetLocker *locker_;
@ -111,8 +113,9 @@ private:
}
return washed_size;
}
} blk_mgr_;
} blk_mgr_, nblk_mgr_;
ObjectSet os_;
ObjectSet nos_;
bool is_inited_;
public:
};
@ -120,6 +123,7 @@ public:
inline ObAllocator::ObAllocator(__MemoryContext__ *mem_context, const ObMemAttr &attr, const bool use_pm,
const uint32_t ablock_size)
: mem_context_(mem_context),
do_not_use_me_(false),
attr_(attr),
use_pm_(use_pm),
pm_(nullptr),
@ -127,8 +131,19 @@ inline ObAllocator::ObAllocator(__MemoryContext__ *mem_context, const ObMemAttr
mutex_(common::ObLatchIds::OB_ALLOCATOR_LOCK),
do_locker_(mutex_),
os_(mem_context_, ablock_size),
nos_(mem_context_, ablock_size),
is_inited_(false)
{
nattr_ = attr_;
do_not_use_me_ = false;
if (FORCE_EXPLICT_500_MALLOC()) {
do_not_use_me_ = OB_SERVER_TENANT_ID == attr.tenant_id_ && !attr.use_500() &&
ob_thread_tenant_id() != OB_SERVER_TENANT_ID;
if (do_not_use_me_) {
nattr_.tenant_id_ = ob_thread_tenant_id();
nattr_.ctx_id_ = ObCtxIds::DO_NOT_USE_ME;
}
}
}
inline int ObAllocator::init()
@ -136,6 +151,7 @@ inline int ObAllocator::init()
int ret = OB_SUCCESS;
ObPageManager *pm = nullptr;
lib::IBlockMgr *blk_mgr = nullptr;
lib::IBlockMgr *nblk_mgr = nullptr;
if (is_inited_) {
ret = OB_INIT_TWICE;
OB_LOG(ERROR, "init twice", K(ret));
@ -149,10 +165,17 @@ inline int ObAllocator::init()
blk_mgr_.set_tenant_id(attr_.tenant_id_);
blk_mgr_.set_ctx_id(attr_.ctx_id_);
blk_mgr = &blk_mgr_;
if (do_not_use_me_) {
nblk_mgr_.set_tenant_id(nattr_.tenant_id_);
nblk_mgr_.set_ctx_id(nattr_.ctx_id_);
nblk_mgr = &nblk_mgr_;
}
}
if (OB_SUCC(ret)) {
os_.set_block_mgr(blk_mgr);
os_.set_locker(locker_);
nos_.set_block_mgr(nblk_mgr);
nos_.set_locker(locker_);
is_inited_ = true;
}
return ret;

16
deps/oblib/src/lib/future/ob_future.h vendored
View File

@ -13,19 +13,19 @@
/**
* ObFuture/ObPromise
*
* ObPromise is a promise of "the result(type is specified by template arg T) will be written and
* ObPromise is a promise of "the result(type is specified by template arg T) will be written and
* only be written once".
* ObFuture is a read-only handle of ObPromise, once result is written, user could read the result
* from ObFuture any number of times, before that, all read operation will hung until the written
* from ObFuture any number of times, before that, all read operation will hung until the written
* done. User could use ObFuture check the result is written or not at any time.
*
*
*
* When to use:
* - you want to synchronize between threads, usually could be described that thread A waiting for
* a operation which will be done by thread B in the future.
* a operation which will be done by thread B in the future.
*
* When not to use:
* - you are crazy about performance, value copy of result is not accepted and condition variable
* - you are crazy about performance, value copy of result is not accepted and condition variable
* is too heavy to you.
*
* Memory usage:
@ -54,8 +54,8 @@
* ObFuture
* 1. int get(T *&ptr) const
* get the promised result, will hung if result is not ready.
* if OB_SUCCESS returned, ptr will pointer to the result, notice that ObFuture is also a
* life time guard of the result, is you do not hold the ObFuture object, do not use ptr get
* if OB_SUCCESS returned, ptr will pointer to the result, notice that ObFuture is also a
* life time guard of the result, is you do not hold the ObFuture object, do not use ptr get
* from this method, althrough it was returned OB_SUCCESS.
* err:
* - OTHERS : forward from ObThreadCond
@ -109,7 +109,7 @@ struct DefaultFutureAllocator : public ObIAllocator {
#ifdef UNIITTEST_DEBUG
total_alive_num++;
#endif
return ob_malloc(size, "ObFuture");
return ob_malloc(size, SET_USE_500("ObFuture"));
}
void* alloc(const int64_t size, const ObMemAttr &attr) override {
UNUSED(attr);

1
deps/oblib/src/rpc/obrpc/ob_poc_rpc_server.cpp vendored
View File

@ -285,6 +285,7 @@ bool ObPocRpcServer::client_use_pkt_nio() {
extern "C" {
void* pkt_nio_malloc(int64_t sz, const char* label) {
ObMemAttr attr(OB_SERVER_TENANT_ID, label, ObCtxIds::PKT_NIO);
SET_USE_500(attr);
return oceanbase::common::ob_malloc(sz, attr);
}
void pkt_nio_free(void *ptr) {

4
src/observer/ob_server_reload_config.cpp
View File

@ -239,6 +239,10 @@ int ObServerReloadConfig::operator()()
}
}
#ifndef ENABLE_SANITY
{
ObMallocAllocator::get_instance()->force_explict_500_malloc_ =
GCONF._force_explict_500_malloc;
}
#else
{
sanity_set_whitelist(GCONF.sanity_whitelist.str());

2
src/observer/omt/ob_tenant_config_mgr.cpp
View File

@ -283,7 +283,7 @@ int ObTenantConfigMgr::add_tenant_config(uint64_t tenant_id)
}
} else {
ObTenantConfig *new_config = nullptr;
new_config = OB_NEW(ObTenantConfig, "TenantConfig", tenant_id);
new_config = OB_NEW(ObTenantConfig, SET_USE_500("TenantConfig"), tenant_id);
if (OB_NOT_NULL(new_config)) {
if(OB_FAIL(new_config->init(this))) {
LOG_WARN("new tenant config init failed", K(ret));

2
src/share/ob_event_history_table_operator.cpp
View File

@ -349,7 +349,7 @@ int ObEventHistoryTableOperator::add_event_to_timer_(const common::ObSqlString &
ObAddr self_addr = self_addr_;
common::ObMySQLProxy *proxy = proxy_;
ObUniqueGuard<ObStringHolder> uniq_holder;
if (OB_FAIL(ob_make_unique(uniq_holder))) {
if (OB_FAIL(ob_make_unique(uniq_holder, SET_USE_500("EventReHolder")))) {
SHARE_LOG(WARN, "fail to make unique guard");
} else if (OB_FAIL(uniq_holder->assign(sql.string()))) {
SHARE_LOG(WARN, "fail to create unique ownership of string");

3
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1493,3 +1493,6 @@ DEF_CAP(_rebuild_replica_log_lag_threshold, OB_TENANT_PARAMETER, "0M", "[0M,)",
DEF_BOOL(_enable_in_range_optimization, OB_TENANT_PARAMETER, "False",
"Enable extract query range optimization for in predicate",
ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_BOOL(_force_explict_500_malloc, OB_CLUSTER_PARAMETER, "False",
"Force 500 memory for explicit allocation",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));

4
src/share/rc/ob_tenant_base.cpp
View File

@ -22,10 +22,6 @@ namespace oceanbase
{
namespace lib
{
int64_t mtl_id()
{
return MTL_CTX() != nullptr && MTL_ID() != 0 ? MTL_ID() : OB_SERVER_TENANT_ID;
}
bool mtl_is_mini_mode()
{
return MTL_CTX() != nullptr && MTL_IS_MINI_MODE();

4
src/sql/session/ob_sql_session_mgr.cpp
View File

@ -274,8 +274,8 @@ int ObSQLSessionMgr::init()
} else if (OB_FAIL(sessid_sequence_.init(MAX_LOCAL_SEQ))) {
LOG_WARN("init sessid sequence failed", K(ret));
} else if (OB_FAIL(sess_hold_map_.create(BUCKET_COUNT,
"SessHoldMapBuck",
"SessHoldMapNode"))) {
SET_USE_500("SessHoldMapBuck"),
SET_USE_500("SessHoldMapNode")))) {
LOG_WARN("failed to init sess_hold_map", K(ret));
}
for (uint32_t i = 1; OB_SUCC(ret) && i <= MAX_LOCAL_SEQ; ++i) {

1
src/storage/tx_storage/ob_tenant_memory_printer.cpp
View File

@ -106,7 +106,6 @@ int ObTenantMemoryPrinter::print_tenant_usage()
if (is_deleted_tenant) {
mallocator->print_tenant_memory_usage(id);
mallocator->print_tenant_ctx_memory_usage(id);
mallocator->print_malloc_sample(id);
}
}
}

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -284,6 +284,7 @@ _enable_trace_session_leak
_enable_transaction_internal_routing
_fast_commit_callback_count
_follower_snapshot_read_retry_duration
_force_explict_500_malloc
_force_hash_groupby_dump
_force_hash_join_spill
_force_skip_encoding_partition_id