hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] Modify lower bound of memory_limit, __easy_memory_limit.

Browse Source
This commit is contained in:
obdev 2023-04-17 13:51:07 +00:00 committed by ob-robot
parent ce5c45de9e
commit f691d3d88f
9 changed files with 66 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
deps/oblib/src/lib/ob_define.h vendored
View File

@ -791,7 +791,8 @@ const int64_t OB_MAX_DDL_SINGLE_REPLICA_BUILD_TIMEOUT = 7L * 24L * 60L * 60L * 1
const char *const OB_STANDBY_USER_NAME = "__oceanbase_inner_standby_user";
const double TENANT_RESERVE_MEM_RATIO = 0.1;
const int64_t LEAST_MEMORY_SIZE = 2L << 30;
const int64_t LEAST_MEMORY_SIZE = 1L << 30;
const int64_t LEAST_MEMORY_SIZE_FOR_NORMAL_MODE = 2L << 30;
const int64_t SYS_MAX_ALLOCATE_MEMORY = 1L << 34;
// mem factor

1
deps/oblib/src/lib/ob_running_mode.cpp vendored
View File

@ -15,6 +15,7 @@
namespace oceanbase {
namespace lib {
const int64_t ObRunningModeConfig::MIN_MEM = 1L << 30; // The minimum value for memory_limit.
const int64_t ObRunningModeConfig::MINI_MEM_LOWER = 4L << 30;
const int64_t ObRunningModeConfig::MINI_MEM_UPPER = 12L << 30;
} //end of namespace lib

1
deps/oblib/src/lib/ob_running_mode.h vendored
View File

@ -20,6 +20,7 @@ namespace lib
{
struct ObRunningModeConfig
{
static const int64_t MIN_MEM;
static const int64_t MINI_MEM_LOWER;
static const int64_t MINI_MEM_UPPER;
bool mini_mode_ = false;

6
src/observer/ob_server.cpp
View File

@ -1624,11 +1624,11 @@ int ObServer::init_pre_setting()
const int64_t reserved_memory = std::min(config_.cache_wash_threshold.get_value(),
static_cast<int64_t>(static_cast<double>(limit_memory) * KVCACHE_FACTOR));
const int64_t reserved_urgent_memory = config_.memory_reserved;
if (!is_arbitration_mode()
&& LEAST_MEMORY_SIZE >= limit_memory) {
const int64_t need_memory_size = is_arbitration_mode() ? LEAST_MEMORY_SIZE : LEAST_MEMORY_SIZE_FOR_NORMAL_MODE;
if (need_memory_size > limit_memory) {
ret = OB_INVALID_CONFIG;
LOG_ERROR("memory limit for oceanbase isn't sufficient",
"need", LEAST_MEMORY_SIZE,
K(need_memory_size),
"limit to", limit_memory,
"sys mem", get_phy_mem_size(),
K(reserved_memory),

2
src/observer/ob_server.h
View File

@ -239,6 +239,7 @@ public:
sql::ObConnectResourceMgr& get_conn_res_mgr() { return conn_res_mgr_; }
obrpc::ObTableRpcProxy &get_table_rpc_proxy() { return table_rpc_proxy_; }
share::ObLocationService &get_location_service() { return location_service_; }
bool is_arbitration_mode() const;
private:
int stop();
@ -300,7 +301,6 @@ private:
int parse_mode();
// ------------------------------- arb server start ------------------------------------
bool is_arbitration_mode() const;
int start_sig_worker_and_handle();
int init_server_in_arb_mode();
int start_server_in_arb_mode();

67
src/observer/ob_server_reload_config.cpp
View File

@ -52,6 +52,7 @@ int set_cluster_name_hash(const ObString &cluster_name)
LOG_WARN("failed to calc_cluster_name_hash", KR(ret), K(cluster_name));
} else {
obrpc::ObRpcNetHandler::CLUSTER_NAME_HASH = cluster_name_hash;
LOG_INFO("set cluster_name_hash", KR(ret), K(cluster_name), K(cluster_name_hash));
}
return ret;
}
@ -88,6 +89,7 @@ int ObServerReloadConfig::operator()()
{
int ret = OB_SUCCESS;
int real_ret = ret;
const bool is_arbitration_mode = OBSERVER.is_arbitration_mode();
if (!gctx_.is_inited()) {
real_ret = ret = OB_INNER_STAT_ERROR;
@ -155,27 +157,30 @@ int ObServerReloadConfig::operator()()
ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval,
GCONF._min_malloc_sample_interval);
#endif
ObIOConfig io_config;
int64_t cpu_cnt = GCONF.cpu_count;
if (cpu_cnt <= 0) {
cpu_cnt = common::get_cpu_num();
}
io_config.disk_io_thread_count_ = GCONF.disk_io_thread_count;
// In the 2.x version, reuse the sys_bkgd_io_timeout configuration item to indicate the data disk io timeout time
// After version 3.1, use the data_storage_io_timeout configuration item.
io_config.data_storage_io_timeout_ms_ = GCONF._data_storage_io_timeout / 1000L;
io_config.data_storage_warning_tolerance_time_ = GCONF.data_storage_warning_tolerance_time;
io_config.data_storage_error_tolerance_time_ = GCONF.data_storage_error_tolerance_time;
if (OB_FAIL(ObIOManager::get_instance().set_io_config(io_config))) {
real_ret = ret;
LOG_WARN("reload io manager config fail, ", K(ret));
}
if (!is_arbitration_mode) {
ObIOConfig io_config;
int64_t cpu_cnt = GCONF.cpu_count;
if (cpu_cnt <= 0) {
cpu_cnt = common::get_cpu_num();
}
io_config.disk_io_thread_count_ = GCONF.disk_io_thread_count;
// In the 2.x version, reuse the sys_bkgd_io_timeout configuration item to indicate the data disk io timeout time
// After version 3.1, use the data_storage_io_timeout configuration item.
io_config.data_storage_io_timeout_ms_ = GCONF._data_storage_io_timeout / 1000L;
io_config.data_storage_warning_tolerance_time_ = GCONF.data_storage_warning_tolerance_time;
io_config.data_storage_error_tolerance_time_ = GCONF.data_storage_error_tolerance_time;
if (!is_arbitration_mode
&& OB_FAIL(ObIOManager::get_instance().set_io_config(io_config))) {
real_ret = ret;
LOG_WARN("reload io manager config fail, ", K(ret));
}
(void)reload_diagnose_info_config(GCONF.enable_perf_event);
(void)reload_trace_log_config(GCONF.enable_record_trace_log);
(void)reload_diagnose_info_config(GCONF.enable_perf_event);
(void)reload_trace_log_config(GCONF.enable_record_trace_log);
reload_tenant_freezer_config_();
reload_tenant_scheduler_config_();
reload_tenant_freezer_config_();
reload_tenant_scheduler_config_();
}
}
{
ObMallocAllocator *malloc_allocator = ObMallocAllocator::get_instance();
@ -190,14 +195,18 @@ int ObServerReloadConfig::operator()()
const int64_t cache_size = GCONF.memory_chunk_cache_size;
const int cache_cnt = (cache_size > 0 ? cache_size : GMEMCONF.get_server_memory_limit()) / INTACT_ACHUNK_SIZE;
lib::AChunkMgr::instance().set_max_chunk_cache_cnt(cache_cnt);
if (GCONF.cluster_id.get_value() >= 0) {
obrpc::ObRpcNetHandler::CLUSTER_ID = GCONF.cluster_id.get_value();
LOG_INFO("set CLUSTER_ID for rpc", "cluster_id", GCONF.cluster_id.get_value());
}
if (FAILEDx(set_cluster_name_hash(GCONF.cluster.str()))) {
LOG_WARN("failed to set_cluster_name_hash", KR(ret), "cluster_name", GCONF.cluster.str(),
"cluster_name_len", strlen(GCONF.cluster.str()));
if (!is_arbitration_mode) {
// Refresh cluster_id, cluster_name_hash for non arbitration mode
if (GCONF.cluster_id.get_value() > 0) {
obrpc::ObRpcNetHandler::CLUSTER_ID = GCONF.cluster_id.get_value();
LOG_INFO("set CLUSTER_ID for rpc", "cluster_id", GCONF.cluster_id.get_value());
}
if (FAILEDx(set_cluster_name_hash(GCONF.cluster.str()))) {
LOG_WARN("failed to set_cluster_name_hash", KR(ret), "cluster_name", GCONF.cluster.str(),
"cluster_name_len", strlen(GCONF.cluster.str()));
}
}
// reset mem leak
@ -250,7 +259,7 @@ int ObServerReloadConfig::operator()()
obrpc::set_rpc_checksum_check_level(new_level);
}
{
if (!is_arbitration_mode) {
auto new_upgrade_stage = obrpc::get_upgrade_stage(GCONF._upgrade_stage.str());
auto orig_upgrade_stage = GCTX.get_upgrade_stage();
if (new_upgrade_stage != orig_upgrade_stage) {
@ -267,8 +276,8 @@ int ObServerReloadConfig::operator()()
lib::g_runtime_enabled = true;
common::ObKVGlobalCache::get_instance().reload_wash_interval();
{
if (!is_arbitration_mode) {
common::ObKVGlobalCache::get_instance().reload_wash_interval();
int tmp_ret = OB_SUCCESS;
int64_t data_disk_size = 0;
int64_t data_disk_percentage = 0;

2
src/share/config/ob_config_helper.cpp
View File

@ -345,7 +345,7 @@ bool ObConfigMemoryLimitChecker::check(const ObConfigItem &t) const
bool is_valid = false;
int64_t value = ObConfigCapacityParser::get(t.str(), is_valid);
if (is_valid) {
is_valid = 0 == value || value >= lib::ObRunningModeConfig::instance().MINI_MEM_LOWER;
is_valid = 0 == value || value >= lib::ObRunningModeConfig::instance().MIN_MEM;
}
return is_valid;
}

19
src/share/config/ob_server_config.cpp
View File

@ -21,6 +21,7 @@
#include "lib/net/ob_net_util.h"
#include "common/ob_record_header.h"
#include "common/ob_zone.h"
#include "observer/ob_server.h"
#include "share/ob_dml_sql_splicer.h"
#include "share/inner_table/ob_inner_table_schema.h"
#include "share/unit/ob_unit_resource.h" // ObUnitResource
@ -215,15 +216,26 @@ ObServerMemoryConfig &ObServerMemoryConfig::get_instance()
int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config)
{
int ret = OB_SUCCESS;
const bool is_arbitration_mode = OBSERVER.is_arbitration_mode();
int64_t memory_limit = server_config.memory_limit;
if (0 == memory_limit) {
memory_limit = get_phy_mem_size() * server_config.memory_limit_percentage / 100;
}
int64_t system_memory = server_config.system_memory;
bool is_valid_config = true;
if (0 == system_memory) {
int64_t memory_limit_g = memory_limit >> 30;
if (memory_limit_g < 4) {
LOG_ERROR("memory_limit with unexpected value", K(memory_limit));
if (memory_limit_g < 1) {
// memory_limit should not be less than 1G for arbitration mode
is_valid_config = false;
LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size());
} else if (is_arbitration_mode) {
// keep system_memory = 0 for arbitration mode
} else if (memory_limit_g < 4) {
// memory_limit should not be less than 4G for non arbitration mode
is_valid_config = false;
LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size());
LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size());
} else if (memory_limit_g <= 8) {
system_memory = 2LL << 30;
} else if (memory_limit_g <= 16) {
@ -238,7 +250,8 @@ int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config)
system_memory = int64_t(15 + 3 * (sqrt(memory_limit_g) - 8)) << 30;
}
}
if (memory_limit > system_memory) {
if (!is_valid_config) {
} else if (memory_limit > system_memory) {
memory_limit_ = memory_limit;
system_memory_ = system_memory;
LOG_INFO("update memory_limit or system_memory success",

6
src/share/parameter/ob_parameter_seed.ipp
View File

@ -77,7 +77,7 @@ DEF_INT(tenant_task_queue_size, OB_CLUSTER_PARAMETER, "16384", "[1024,]",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
_DEF_PARAMETER_SCOPE_CHECKER_EASY(private, Capacity, memory_limit, OB_CLUSTER_PARAMETER, "0",
common::ObConfigMemoryLimitChecker, "[0M,)",
"the size of the memory reserved for internal use(for testing purpose), 0 means follow memory_limit_percentage. Range: 0, [4G,).",
"the size of the memory reserved for internal use(for testing purpose), 0 means follow memory_limit_percentage. Range: 0, [1G,).",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_CAP(rootservice_memory_limit, OB_CLUSTER_PARAMETER, "2G", "[2G,)",
"max memory size which can be used by rs tenant The default value is 2G. Range: [2G,)",
@ -976,8 +976,8 @@ DEF_INT(module_test_trx_memory_errsim_percentage, OB_CLUSTER_PARAMETER, "0", "[0
DEF_CAP(sql_work_area, OB_TENANT_PARAMETER, "1G", "[10M,)",
"Work area memory limitation for tenant",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::STATIC_EFFECTIVE));
DEF_CAP(__easy_memory_limit, OB_CLUSTER_PARAMETER, "4G", "[1G,)",
"max memory size which can be used by libeasy. The default value is 4G. Range: [1G,)",
DEF_CAP(__easy_memory_limit, OB_CLUSTER_PARAMETER, "4G", "[256M,)",
"max memory size which can be used by libeasy. The default value is 4G. Range: [256M,)",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_CAP(stack_size, OB_CLUSTER_PARAMETER, "512K", "[512K, 20M]",
"the size of routine execution stack"