hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

avoid to madvise the huge pages memory

Browse Source
This commit is contained in:
tushicheng 2024-06-18 01:03:19 +00:00 committed by ob-robot
parent 789be4de37
commit a79ae9405f
7 changed files with 49 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
deps/oblib/src/lib/alloc/alloc_failed_reason.cpp vendored
View File

@ -98,16 +98,21 @@ char *alloc_failed_msg()
}
case PHYSICAL_MEMORY_EXHAUST: {
int64_t process_hold = 0;
get_process_physical_hold(process_hold);
int64_t virtual_memory_used = common::get_virtual_memory_used(&process_hold);
snprintf(msg, len,
"physical memory exhausted(os_total: %ld, os_available: %ld, server_hold: %ld, errno: %d, alloc_size: %ld)",
"physical memory exhausted(os_total: %ld, os_available: %ld, virtual_memory_used: %ld, server_hold: %ld, errno: %d, alloc_size: %ld)",
sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE),
sysconf(_SC_AVPHYS_PAGES) * sysconf(_SC_PAGESIZE),
virtual_memory_used,
process_hold,
afc.errno_,
afc.alloc_size_);
break;
}
case ERRSIM: {
snprintf(msg, len, "errsim");
break;
}
default: {
snprintf(msg, len, "unknown reason");
break;

3
deps/oblib/src/lib/alloc/alloc_failed_reason.h vendored
View File

@ -27,7 +27,8 @@ enum AllocFailedReason
CTX_HOLD_REACH_LIMIT,
TENANT_HOLD_REACH_LIMIT,
SERVER_HOLD_REACH_LIMIT,
PHYSICAL_MEMORY_EXHAUST
PHYSICAL_MEMORY_EXHAUST,
ERRSIM
};
struct AllocFailedCtx

20
deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp vendored
View File

@ -17,7 +17,6 @@
#include "lib/alloc/object_set.h"
#include "lib/alloc/memory_sanity.h"
#include "lib/alloc/memory_dump.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/allocator/ob_mem_leak_checker.h"
#include "lib/allocator/ob_page_manager.h"
#include "lib/rc/ob_rc.h"
@ -102,7 +101,7 @@ void *ObMallocAllocator::alloc(const int64_t size, const oceanbase::lib::ObMemAt
#else
SANITY_DISABLE_CHECK_RANGE(); // prevent sanity_check_range
ObDisableDiagnoseGuard disable_diagnose_guard;
int ret = OB_E(EventTable::EN_4) OB_SUCCESS;
int ret = OB_SUCCESS;
void *ptr = NULL;
ObTenantCtxAllocatorGuard allocator = NULL;
lib::ObMemAttr attr = _attr;
@ -184,23 +183,10 @@ void *ObMallocAllocator::realloc(
ObDisableDiagnoseGuard disable_diagnose_guard;
// Won't create tenant allocator!!
void *nptr = NULL;
int ret = OB_E(EventTable::EN_4) OB_SUCCESS;
if (NULL != ptr) {
AObject *obj = reinterpret_cast<AObject*>((char*)ptr - AOBJECT_HEADER_SIZE);
abort_unless(NULL != obj);
abort_unless(obj->MAGIC_CODE_ == AOBJECT_MAGIC_CODE
|| obj->MAGIC_CODE_ == BIG_AOBJECT_MAGIC_CODE);
abort_unless(obj->in_use_);
get_mem_leak_checker().on_free(*obj);
}
oceanbase::lib::ObMemAttr inner_attr = attr;
ObTenantCtxAllocatorGuard allocator = NULL;
if (OB_FAIL(ret) && NULL == ptr) {
if (OB_ISNULL(allocator = get_tenant_ctx_allocator(attr.tenant_id_, attr.ctx_id_))) {
// do nothing
} else if (OB_ISNULL(allocator = get_tenant_ctx_allocator(inner_attr.tenant_id_, inner_attr.ctx_id_))) {
// do nothing
} else if (OB_ISNULL(nptr = allocator->realloc(ptr, size, inner_attr))) {
} else if (OB_ISNULL(nptr = allocator->realloc(ptr, size, attr))) {
// do nothing
}
return nptr;

14
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -18,6 +18,7 @@
#include "lib/allocator/ob_mem_leak_checker.h"
#include "lib/allocator/ob_tc_malloc.h"
#include "lib/utility/ob_print_utils.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/alloc/memory_dump.h"
#include "lib/alloc/memory_sanity.h"
#include "lib/alloc/ob_malloc_callback.h"
@ -482,7 +483,6 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
obj = reinterpret_cast<AObject*>((char*)ptr - AOBJECT_HEADER_SIZE);
on_free(*obj);
}
#ifdef ERRSIM
const ObErrsimModuleType type = THIS_WORKER.get_module_type();
if (is_errsim_module(ta.get_tenant_id(), type.type_)) {
@ -494,7 +494,15 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
ObLightBacktraceGuard light_backtrace_guard(is_memleak_light_backtrace_enabled()
&& ObCtxIds::GLIBC != attr.ctx_id_);
ObMemAttr inner_attr = attr;
if (OB_UNLIKELY(is_errsim)) {
int ret = OB_E(EventTable::EN_4) OB_SUCCESS;
if (OB_UNLIKELY(OB_FAIL(ret) || is_errsim)) {
AllocFailedCtx &afc = g_alloc_failed_ctx();
afc.reason_ = AllocFailedReason::ERRSIM;
if (OB_NOT_NULL(obj)) {
allocator.free_object(obj);
obj = NULL;
}
} else {
BASIC_TIME_GUARD(time_guard, "ObMalloc");
DEFER(ObMallocTimeMonitor::get_instance().record_malloc_time(time_guard, size, inner_attr));
@ -527,7 +535,7 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
sleep(1);
}
#endif
const char *msg = is_errsim ? "[ERRSIM] errsim inject memory error" : alloc_failed_msg();
const char *msg = alloc_failed_msg();
LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg));
_OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, "
"ctx_limit=%ld, tenant_hold=%ld, tenant_limit=%ld",

4
deps/oblib/src/lib/alloc/object_mgr.cpp vendored
View File

@ -256,6 +256,10 @@ SubObjectMgr *ObjectMgr::create_sub_mgr()
sub_mgr_.unlock();
return new_obj;
}
void free_object(AObject *obj)
{
sub_mgr_.free_object(obj);
}
private:
SubObjectMgr& sub_mgr_;
} root_mgr(static_cast<ObjectMgr&>(ta->get_block_mgr()).root_mgr_);

32
deps/oblib/src/lib/resource/achunk_mgr.cpp vendored
View File

@ -174,7 +174,7 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo
ssize_t shad_size = SANITY_TO_SHADOW_SIZE(size);
if (MAP_FAILED == ::mmap(shad_ptr, shad_size, prot, flags, fd, offset)) {
LOG_WARN_RET(OB_ALLOCATE_MEMORY_FAILED, "sanity alloc shadow failed", K(errno), KP(shad_ptr));
::munmap(ptr, size);
this->munmap(ptr, size);
ptr = nullptr;
} else {
IGNORE_RETURN ATOMIC_FAA(&shadow_hold_, shad_size);
@ -194,9 +194,9 @@ void AChunkMgr::low_free(const void *ptr, const uint64_t size)
void *shad_ptr = SANITY_TO_SHADOW((void*)ptr);
ssize_t shad_size = SANITY_TO_SHADOW_SIZE(size);
IGNORE_RETURN ATOMIC_FAA(&shadow_hold_, -shad_size);
::munmap(shad_ptr, shad_size);
this->munmap(shad_ptr, shad_size);
}
::munmap((void*)ptr, size);
this->munmap((void*)ptr, size);
unset_ob_mem_mgr_path();
}
@ -209,25 +209,27 @@ AChunk *AChunkMgr::alloc_chunk(const uint64_t size, bool high_prio)
// Reuse chunk from self-cache
if (OB_NOT_NULL(chunk = pop_chunk_with_size(all_size))) {
int64_t orig_hold_size = chunk->hold();
bool need_free = false;
if (hold_size == orig_hold_size) {
// do-nothing
} else if (hold_size > orig_hold_size) {
if (!update_hold(hold_size - orig_hold_size, high_prio)) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
}
need_free = !update_hold(hold_size - orig_hold_size, high_prio);
} else if (chunk->is_hugetlb_) {
need_free = true;
} else {
int result = this->madvise((char*)chunk + hold_size, orig_hold_size - hold_size, MADV_DONTNEED);
if (-1 == result) {
LOG_WARN_RET(OB_ERR_SYS, "madvise failed", K(errno));
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
need_free = true;
} else {
IGNORE_RETURN update_hold(hold_size - orig_hold_size, false);
}
}
if (need_free) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
}
}
if (OB_ISNULL(chunk)) {
bool updated = false;
@ -371,6 +373,14 @@ int AChunkMgr::madvise(void *addr, size_t length, int advice)
return result;
}
void AChunkMgr::munmap(void *addr, size_t length)
{
int orig_errno = errno;
if (-1 == ::munmap(addr, length)) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "munmap failed", KP(addr), K(length), K(orig_errno), K(errno));
}
}
int64_t AChunkMgr::to_string(char *buf, const int64_t buf_len) const
{
int ret = OB_SUCCESS;

1
deps/oblib/src/lib/resource/achunk_mgr.h vendored
View File

@ -238,6 +238,7 @@ public:
inline static AChunk *ptr2chunk(const void *ptr);
bool update_hold(int64_t bytes, bool high_prio);
virtual int madvise(void *addr, size_t length, int advice);
void munmap(void *addr, size_t length);
int64_t to_string(char *buf, const int64_t buf_len) const;
inline void set_limit(int64_t limit);