hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

4G mysqltest

Browse Source
This commit is contained in:
tushicheng
2023-09-27 09:14:01 +00:00
committed by ob-robot
parent c6f228ce59
commit ba2ac56c8e
12 changed files with 103 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
deps/oblib/src/lib/alloc/ob_malloc_allocator.cpp vendored
View File

@ -411,43 +411,6 @@ int ObMallocAllocator::with_resource_handle_invoke(uint64_t tenant_id, InvokeFun
}
return ret;
}
#ifdef ENABLE_500_MEMORY_LIMIT
int ObMallocAllocator::set_500_tenant_limit(const bool unlimited)
{
int ret = OB_SUCCESS;
for (int ctx_id = 0; OB_SUCC(ret) && ctx_id < ObCtxIds::MAX_CTX_ID; ++ctx_id) {
if (ObCtxIds::SCHEMA_SERVICE == ctx_id ||
ObCtxIds::PKT_NIO == ctx_id ||
ObCtxIds::CO_STACK == ctx_id ||
ObCtxIds::LIBEASY == ctx_id ||
ObCtxIds::GLIBC == ctx_id ||
ObCtxIds::LOGGER_CTX_ID== ctx_id ||
ObCtxIds::RPC_CTX_ID == ctx_id ||
ObCtxIds::UNEXPECTED_IN_500 == ctx_id) {
continue;
}
auto ta = get_tenant_ctx_allocator(OB_SERVER_TENANT_ID, ctx_id);
if (OB_NOT_NULL(ta)) {
int64_t ctx_limit = ObCtxIds::DEFAULT_CTX_ID == ctx_id ? (4LL<<30) : (50LL<<20);
if (unlimited) {
ctx_limit = INT64_MAX;
}
if (OB_FAIL(ta->set_limit(ctx_limit))) {
LIB_LOG(WARN, "set limit of 500 tenant failed", K(ret), K(ctx_limit),
"ctx_name", get_global_ctx_info().get_ctx_name(ctx_id));
} else {
LIB_LOG(INFO, "set limit of 500 tenant succeed", K(ret), K(ctx_limit),
"ctx_name", get_global_ctx_info().get_ctx_name(ctx_id));
}
} else {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "tenant ctx allocator is not exist", K(ret),
"ctx_name", get_global_ctx_info().get_ctx_name(ctx_id));
}
}
return ret;
}
#endif
int ObMallocAllocator::set_tenant_limit(uint64_t tenant_id, int64_t bytes)
{

3
deps/oblib/src/lib/alloc/ob_malloc_allocator.h vendored
View File

@ -122,9 +122,6 @@ public:
int64_t get_urgent() const;
void set_reserved(int64_t bytes);
int64_t get_reserved() const;
#ifdef ENABLE_500_MEMORY_LIMIT
int set_500_tenant_limit(const bool unlimited);
#endif
int set_tenant_limit(uint64_t tenant_id, int64_t bytes);
int64_t get_tenant_limit(uint64_t tenant_id);
int64_t get_tenant_hold(uint64_t tenant_id);

8
deps/oblib/src/lib/resource/ob_resource_mgr.cpp vendored
View File

@ -27,6 +27,8 @@ namespace oceanbase
using namespace common;
namespace lib
{
bool ObTenantMemoryMgr::error_log_when_tenant_500_oversize = false;
ObTenantMemoryMgr::ObTenantMemoryMgr()
: cache_washer_(NULL), tenant_id_(common::OB_INVALID_ID),
limit_(INT64_MAX), sum_hold_(0), rpc_hold_(0), cache_hold_(0),
@ -265,6 +267,12 @@ bool ObTenantMemoryMgr::update_hold(const int64_t size, const uint64_t ctx_id,
updated = true;
}
}
if (OB_UNLIKELY(error_log_when_tenant_500_oversize &&
OB_SERVER_TENANT_ID == tenant_id_ &&
sum_hold_ > (1LL<<30))) {
LOG_ERROR_RET(OB_ERROR, "the hold memory of tenant_500 is over the reserved memory",
K_(sum_hold));
}
}
if (!updated) {
auto &afc = g_alloc_failed_ctx();

1
deps/oblib/src/lib/resource/ob_resource_mgr.h vendored
View File

@ -31,6 +31,7 @@ class ObTenantMemoryMgr
public:
static const int64_t LARGE_REQUEST_EXTRA_MB_COUNT = 2;
static const int64_t ALIGN_SIZE = static_cast<int64_t>(INTACT_ACHUNK_SIZE);
static bool error_log_when_tenant_500_oversize;
ObTenantMemoryMgr();
ObTenantMemoryMgr(const uint64_t tenant_id);

16
src/diagnose/lua/ob_lua_api.cpp
View File

@ -141,7 +141,7 @@ static constexpr const char *usage_str =
"{{row1}, {row2}, ...} = select_schema_slot()\n"
"{{row1}, {row2}, ...} = dump_thread_info()\n"
"{{row1}, {row2}, ...} = select_malloc_sample_info()\n"
"int = enable_system_tenant_memory_limit(boolean)\n"
"int = set_system_tenant_limit_mode(int)\n"
;
class LuaVtableGenerator
@ -1998,19 +1998,19 @@ int select_malloc_sample_info(lua_State *L)
return 1;
}
// int = enable_system_tenant_memory_limit(boolean)
int enable_system_tenant_memory_limit(lua_State* L)
// int = set_system_tenant_limit_mode(int)
int set_system_tenant_limit_mode(lua_State* L)
{
int ret = OB_SUCCESS;
int argc = lua_gettop(L);
if (1 != argc) {
OB_LOG(ERROR, "call enable_system_tenant_memory_limit() failed, bad arguments count, should be 1.");
OB_LOG(ERROR, "call set_system_tenant_limit_mode() failed, bad arguments count, should be 1.");
ret = OB_INVALID_ARGUMENT;
} else {
luaL_checktype(L, 1, LUA_TBOOLEAN);
int enable = lua_toboolean(L, 1);
luaL_checktype(L, 1, LUA_TNUMBER);
const int64_t limit_mode = lua_tointeger(L, 1);
#ifdef ENABLE_500_MEMORY_LIMIT
ObMallocAllocator::get_instance()->set_500_tenant_limit(!enable/*unlimited*/);
GMEMCONF.set_500_tenant_limit(limit_mode);
#endif
lua_pushinteger(L, 1);
}
@ -2148,7 +2148,7 @@ void APIRegister::register_api(lua_State* L)
lua_register(L, "select_schema_slot", select_schema_slot);
lua_register(L, "dump_thread_info", dump_thread_info);
lua_register(L, "select_malloc_sample_info", select_malloc_sample_info);
lua_register(L, "enable_system_tenant_memory_limit", enable_system_tenant_memory_limit);
lua_register(L, "set_system_tenant_limit_mode", set_system_tenant_limit_mode);
}
int APIRegister::flush()

2
src/diagnose/lua/test.lua
View File

@ -338,4 +338,4 @@ para["select"] = {
print_to_client("select_malloc_sample_info")
select_malloc_sample_info(para)
enable_system_tenant_memory_limit(true)
set_system_tenant_limit_mode(1)

87
src/share/config/ob_server_config.cpp
View File

@ -356,34 +356,91 @@ int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config)
LOG_ERROR("update observer memory config failed",
K(memory_limit), K(system_memory), K(hidden_sys_memory), K(min_server_avail_memory));
}
//check the hold memory of tenant 500
int64_t tenant_500_hold = lib::get_tenant_memory_hold(OB_SERVER_TENANT_ID);
int64_t tenant_500_reserved = system_memory_ - get_extra_memory();
if (tenant_500_hold > tenant_500_reserved) {
if (server_config._ignore_system_memory_over_limit_error) {
LOG_ERROR("the hold memory of tenant_500 is over the reserved memory",
K(tenant_500_hold), K(tenant_500_reserved));
} else {
LOG_WARN("the hold memory of tenant_500 is over the reserved memory",
K(tenant_500_hold), K(tenant_500_reserved));
}
}
}
#ifdef ENABLE_500_MEMORY_LIMIT
if (OB_FAIL(ret)) {
// do-nothing
} else if (is_arbitration_mode) {
// do-nothing
} else if (OB_FAIL(ObMallocAllocator::get_instance()->set_500_tenant_limit(
!server_config._enable_system_tenant_memory_limit))) {
} else if (OB_FAIL(set_500_tenant_limit(server_config._system_tenant_limit_mode))) {
LOG_ERROR("set the limit of tenant 500 failed", KR(ret));
}
#endif
return ret;
}
void ObServerMemoryConfig::check_500_tenant_hold(bool ignore_error)
{
//check the hold memory of tenant 500
int ret = OB_SUCCESS;
int64_t hold = lib::get_tenant_memory_hold(OB_SERVER_TENANT_ID);
int64_t reserved = system_memory_ - get_extra_memory();
if (hold > reserved) {
if (ignore_error) {
LOG_WARN("the hold memory of tenant_500 is over the reserved memory",
K(hold), K(reserved));
} else {
LOG_ERROR("the hold memory of tenant_500 is over the reserved memory",
K(hold), K(reserved));
}
}
}
#ifdef ENABLE_500_MEMORY_LIMIT
int ObServerMemoryConfig::set_500_tenant_limit(const int64_t limit_mode)
{
const int64_t UNLIMIT_MODE = 0;
const int64_t CTX_LIMIT_MODE = 1;
const int64_t TENANT_LIMIT_MODE = 2;
int ret = OB_SUCCESS;
bool unlimited = false;
auto ma = ObMallocAllocator::get_instance();
if (UNLIMIT_MODE == limit_mode) {
unlimited = true;
ObTenantMemoryMgr::error_log_when_tenant_500_oversize = false;
} else if (CTX_LIMIT_MODE == limit_mode) {
ObTenantMemoryMgr::error_log_when_tenant_500_oversize = false;
} else if (TENANT_LIMIT_MODE == limit_mode) {
ObTenantMemoryMgr::error_log_when_tenant_500_oversize = true;
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid limit mode", K(ret), K(limit_mode));
}
for (int ctx_id = 0; OB_SUCC(ret) && ctx_id < ObCtxIds::MAX_CTX_ID; ++ctx_id) {
if (ObCtxIds::SCHEMA_SERVICE == ctx_id ||
ObCtxIds::PKT_NIO == ctx_id ||
ObCtxIds::CO_STACK == ctx_id ||
ObCtxIds::LIBEASY == ctx_id ||
ObCtxIds::GLIBC == ctx_id ||
ObCtxIds::LOGGER_CTX_ID== ctx_id ||
ObCtxIds::RPC_CTX_ID == ctx_id ||
ObCtxIds::UNEXPECTED_IN_500 == ctx_id) {
continue;
}
auto ta = ma->get_tenant_ctx_allocator(OB_SERVER_TENANT_ID, ctx_id);
const char *ctx_name = get_global_ctx_info().get_ctx_name(ctx_id);
if (OB_NOT_NULL(ta)) {
int64_t ctx_limit = ObCtxIds::DEFAULT_CTX_ID == ctx_id ? (4LL<<30) : (50LL<<20);
if (unlimited) {
ctx_limit = INT64_MAX;
}
if (OB_FAIL(ta->set_limit(ctx_limit))) {
LOG_WARN("set ctx limit of 500 tenant failed", K(ret), K(ctx_limit),
"ctx_name", ctx_name);
}
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tenant ctx allocator is not exist", K(ret),
"ctx_name", ctx_name);
}
}
return ret;
}
#endif
int ObServerConfig::serialize_(char *buf, const int64_t buf_len, int64_t &pos) const
{
int ret = OB_SUCCESS;

4
src/share/config/ob_server_config.h
View File

@ -168,7 +168,11 @@ public:
int64_t get_hidden_sys_memory() { return hidden_sys_memory_; }
//the extra_memory just used by real sys when non_mini_mode
int64_t get_extra_memory();
void check_500_tenant_hold(bool ignore_error);
#ifdef ENABLE_500_MEMORY_LIMIT
int set_500_tenant_limit(const int64_t limit_mode);
#endif
private:
int64_t get_adaptive_memory_config(const int64_t memory_size,
DependentMemConfig dep_mem_config,

6
src/share/parameter/ob_parameter_seed.ipp
View File

@ -1603,6 +1603,12 @@ DEF_TIME(_schema_memory_recycle_interval, OB_CLUSTER_PARAMETER, "15m", "[0s,)",
DEF_BOOL(_enable_system_tenant_memory_limit, OB_CLUSTER_PARAMETER, "True",
"specifies whether allowed to limit the memory of tenant 500",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_INT(_system_tenant_limit_mode, OB_CLUSTER_PARAMETER, "1", "[0,2]",
"specifies the limit mode for the memory hold of system tenant, "
"0: not limit the memory hold of system tenant, "
"1: only limit the DEFAULT_CTX_ID memory of system tenant, "
"2: besides limit the DEFAULT_CTX_ID memory, the total hold of system tenant is also limited.",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
#endif
DEF_BOOL(_force_malloc_for_absent_tenant, OB_CLUSTER_PARAMETER, "False",
"force malloc even if tenant does not exist in observer",

2
src/storage/tx_storage/ob_tenant_memory_printer.cpp
View File

@ -28,6 +28,7 @@ namespace storage
{
void ObPrintTenantMemoryUsage::runTimerTask()
{
GMEMCONF.check_500_tenant_hold(GCONF._ignore_system_memory_over_limit_error);
LOG_INFO("=== Run print tenant memory usage task ===");
ObTenantMemoryPrinter &printer = ObTenantMemoryPrinter::get_instance();
printer.print_tenant_usage();
@ -163,7 +164,6 @@ int ObTenantMemoryPrinter::print_tenant_usage()
);
print_mutex_.unlock();
}
return ret;
}

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -377,6 +377,7 @@ _sqlexec_disable_hash_based_distagg_tiv
_sql_insert_multi_values_split_opt
_stall_threshold_for_dynamic_worker
_storage_meta_memory_limit_percentage
_system_tenant_limit_mode
_temporary_file_io_area_size
_trace_control_info
_transfer_finish_trans_timeout

1
tools/deploy/obd/core-test.yaml.template
View File

@ -7,4 +7,5 @@ oceanbase-ce:
system_memory: '1G'
datafile_size: '20G'
cpu_count: '24'
_system_tenant_limit_mode: '2'
{{%% PROXY_CONF %%}}