BUGFIX: change memstore_limit to tenant level

This commit is contained in:
obdev
2024-01-04 12:13:24 +00:00
committed by ob-robot
parent 61f14781c7
commit 7f1ad4aa49
18 changed files with 151 additions and 45 deletions

View File

@ -28,6 +28,7 @@
#include "observer/ob_server.h"
#include "observer/ob_server_utils.h"
#include "observer/ob_service.h"
#include "share/allocator/ob_shared_memory_allocator_mgr.h"
#include "storage/tx_storage/ob_tenant_freezer.h"
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
#include "storage/slog/ob_storage_logger_manager.h"
@ -169,6 +170,7 @@ int ObServerReloadConfig::operator()()
(void)reload_diagnose_info_config(GCONF.enable_perf_event);
(void)reload_trace_log_config(GCONF.enable_record_trace_log);
reload_tenant_freezer_config_();
reload_tenant_scheduler_config_();
}
}
@ -336,3 +338,25 @@ void ObServerReloadConfig::reload_tenant_scheduler_config_()
omt->operate_in_each_tenant(f);
}
}
int ObServerReloadConfig::ObReloadTenantFreezerConfOp::operator()()
{
int ret = OB_SUCCESS;
// NOTICE: tenant freezer should update before ObSharedMemAllocMgr.
MTL(ObTenantFreezer *)->reload_config();
MTL(ObSharedMemAllocMgr*)->update_throttle_config();
return ret;
}
void ObServerReloadConfig::reload_tenant_freezer_config_()
{
int ret = OB_SUCCESS;
omt::ObMultiTenant *omt = GCTX.omt_;
if (OB_ISNULL(omt)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("omt should not be null", K(ret));
} else {
ObReloadTenantFreezerConfOp f;
omt->operate_in_each_tenant(f);
}
}

View File

@ -32,10 +32,14 @@ public:
virtual ~ObServerReloadConfig();
int operator()();
class ObReloadTenantFreezerConfOp
{
public:
int operator()();
};
private:
void reload_tenant_scheduler_config_();
void reload_tenant_freezer_config_();
private:
ObGlobalContext &gctx_;
};

View File

@ -9965,6 +9965,8 @@ int ObRootService::set_config_pre_hook(obrpc::ObAdminSetConfigArg &arg)
ret = check_tx_share_memory_limit_(*item);
} else if (0 == STRCMP(item->name_.ptr(), MEMSTORE_LIMIT_PERCENTAGE)) {
ret = check_memstore_limit_(*item);
} else if (0 == STRCMP(item->name_.ptr(), TENANT_MEMSTORE_LIMIT_PERCENTAGE)) {
ret = check_tenant_memstore_limit_(*item);
} else if (0 == STRCMP(item->name_.ptr(), _TX_DATA_MEMORY_LIMIT_PERCENTAGE)) {
ret = check_tx_data_memory_limit_(*item);
} else if (0 == STRCMP(item->name_.ptr(), _MDS_MEMORY_LIMIT_PERCENTAGE)) {
@ -10066,6 +10068,19 @@ int ObRootService::set_config_pre_hook(obrpc::ObAdminSetConfigArg &arg)
} \
} while (0)
#define CHECK_CLUSTER_CONFIG_WITH_FUNC(FUNCTOR, LOG_INFO) \
do { \
bool valid = true; \
for (int i = 0; i < tenant_ids.count() && valid; i++) { \
valid = valid && FUNCTOR::check(tenant_ids.at(i), item); \
if (!valid) { \
ret = OB_INVALID_ARGUMENT; \
LOG_USER_ERROR(OB_INVALID_ARGUMENT, LOG_INFO); \
LOG_WARN("config invalid", "item", item, K(ret), K(i), K(tenant_ids.at(i))); \
} \
} \
} while (0)
int ObRootService::check_tx_share_memory_limit_(obrpc::ObAdminSetConfigItem &item)
{
int ret = OB_SUCCESS;
@ -10079,9 +10094,32 @@ int ObRootService::check_tx_share_memory_limit_(obrpc::ObAdminSetConfigItem &ite
int ObRootService::check_memstore_limit_(obrpc::ObAdminSetConfigItem &item)
{
int ret = OB_SUCCESS;
const char *warn_log = "tenant config memstore_limit_percentage. "
const char *warn_log = "cluster config memstore_limit_percentage. "
"It should less than or equal with all tenant's _tx_share_memory_limit_percentage";
ObArray<uint64_t> tenant_ids;
ObSchemaGetterGuard schema_guard;
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", KR(ret));
} else if (OB_ISNULL(schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("schema service is null", KR(ret));
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
LOG_WARN("get schema guard failed", KR(ret));
} else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) {
LOG_WARN("failed to get all tenant ids", KR(ret), K(tenant_ids));
} else {
CHECK_CLUSTER_CONFIG_WITH_FUNC(ObConfigMemstoreLimitChecker, warn_log);
}
return ret;
}
int ObRootService::check_tenant_memstore_limit_(obrpc::ObAdminSetConfigItem &item)
{
int ret = OB_SUCCESS;
const char *warn_log = "tenant config _memstore_limit_percentage. "
"It should less than or equal with _tx_share_memory_limit_percentage";
CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigTxDataLimitChecker, warn_log);
CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigMemstoreLimitChecker, warn_log);
return ret;
}
@ -10122,6 +10160,7 @@ int ObRootService::check_write_throttle_trigger_percentage(obrpc::ObAdminSetConf
}
#undef CHECK_TENANTS_CONFIG_WITH_FUNC
#undef CHECK_CLUSTER_CONFIG_WITH_FUNC
int ObRootService::set_config_post_hook(const obrpc::ObAdminSetConfigArg &arg)
{

View File

@ -905,6 +905,7 @@ private:
int check_tx_share_memory_limit_(obrpc::ObAdminSetConfigItem &item);
int check_memstore_limit_(obrpc::ObAdminSetConfigItem &item);
int check_tenant_memstore_limit_(obrpc::ObAdminSetConfigItem &item);
int check_tx_data_memory_limit_(obrpc::ObAdminSetConfigItem &item);
int check_mds_memory_limit_(obrpc::ObAdminSetConfigItem &item);
int check_freeze_trigger_percentage_(obrpc::ObAdminSetConfigItem &item);

View File

@ -214,7 +214,6 @@ void ObMemstoreAllocator::init_throttle_config(int64_t &resource_limit,
int64_t &max_duration)
{
// define some default value
const int64_t MEMSTORE_LIMIT_PERCENTAGE = 50;
const int64_t MEMSTORE_THROTTLE_TRIGGER_PERCENTAGE = 60;
const int64_t MEMSTORE_THROTTLE_MAX_DURATION = 2LL * 60LL * 60LL * 1000LL * 1000LL; // 2 hours
@ -223,15 +222,14 @@ void ObMemstoreAllocator::init_throttle_config(int64_t &resource_limit,
// Use tenant config to init throttle config
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
resource_limit = total_memory * tenant_config->memstore_limit_percentage / 100LL;
trigger_percentage = tenant_config->writing_throttling_trigger_percentage;
max_duration = tenant_config->writing_throttling_maximum_duration;
} else {
COMMON_LOG_RET(WARN, OB_INVALID_CONFIG, "init throttle config with default value");
resource_limit = total_memory * MEMSTORE_LIMIT_PERCENTAGE / 100;
trigger_percentage = MEMSTORE_THROTTLE_TRIGGER_PERCENTAGE;
max_duration = MEMSTORE_THROTTLE_MAX_DURATION;
}
resource_limit = total_memory * MTL(storage::ObTenantFreezer *)->get_memstore_limit_percentage() / 100;
}
void ObMemstoreAllocator::adaptive_update_limit(const int64_t tenant_id,

View File

@ -40,18 +40,18 @@ void ObSharedMemAllocMgr::update_throttle_config()
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
int64_t share_mem_limit_percentage = tenant_config->_tx_share_memory_limit_percentage;
int64_t memstore_limit_percentage = tenant_config->memstore_limit_percentage;
int64_t tenant_memstore_limit_percentage = MTL(ObTenantFreezer*)->get_memstore_limit_percentage();
int64_t tx_data_limit_percentage = tenant_config->_tx_data_memory_limit_percentage;
int64_t mds_limit_percentage = tenant_config->_mds_memory_limit_percentage;
int64_t trigger_percentage = tenant_config->writing_throttling_trigger_percentage;
int64_t max_duration = tenant_config->writing_throttling_maximum_duration;
if (0 == share_mem_limit_percentage) {
// 0 means use (memstore_limit + 10)
share_mem_limit_percentage = memstore_limit_percentage + 10;
share_mem_limit_percentage = tenant_memstore_limit_percentage + 10;
}
int64_t share_mem_limit = total_memory / 100 * share_mem_limit_percentage;
int64_t memstore_limit = total_memory / 100 * memstore_limit_percentage;
int64_t memstore_limit = total_memory / 100 * tenant_memstore_limit_percentage;
int64_t tx_data_limit = total_memory / 100 * tx_data_limit_percentage;
int64_t mds_limit = total_memory / 100 * mds_limit_percentage;
@ -78,7 +78,7 @@ void ObSharedMemAllocMgr::update_throttle_config()
K(total_memory),
K(share_mem_limit_percentage),
K(share_mem_limit),
K(memstore_limit_percentage),
K(tenant_memstore_limit_percentage),
K(memstore_limit),
K(tx_data_limit_percentage),
K(tx_data_limit),

View File

@ -139,12 +139,8 @@ int ObTenantMutilAllocatorMgr::get_tenant_memstore_limit_percent_(const uint64_t
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid arguments", K(ret), K(tenant_id));
} else {
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
if (!tenant_config.is_valid()) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(ERROR, "tenant config invalid", K(ret), K(tenant_id));
} else {
limit_percent = tenant_config->memstore_limit_percentage;
MTL_SWITCH(tenant_id) {
limit_percent = MTL(ObTenantFreezer*)->get_memstore_limit_percentage();
}
}
return ret;

View File

@ -105,13 +105,14 @@ bool ObConfigTxShareMemoryLimitChecker::check(const uint64_t tenant_id, const Ob
{
bool is_valid = false;
int64_t value = ObConfigIntParser::get(t.value_.ptr(), is_valid);
int64_t cluster_memstore_limit = GCONF.memstore_limit_percentage;
int64_t memstore_limit = 0;
int64_t tx_data_limit = 0;
int64_t mds_limit = 0;
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
if (tenant_config.is_valid()) {
memstore_limit = tenant_config->memstore_limit_percentage;
memstore_limit = tenant_config->_memstore_limit_percentage;
tx_data_limit = tenant_config->_tx_data_memory_limit_percentage;
mds_limit = tenant_config->_mds_memory_limit_percentage;
} else {
@ -119,7 +120,13 @@ bool ObConfigTxShareMemoryLimitChecker::check(const uint64_t tenant_id, const Ob
OB_LOG_RET(ERROR, OB_INVALID_CONFIG, "tenant config is invalid", K(tenant_id));
}
if (0 == memstore_limit) {
memstore_limit = cluster_memstore_limit;
}
if (!is_valid) {
} else if (0 == memstore_limit) {
// both 0 means adjust the percentage automatically.
is_valid = true;
} else if (0 == value) {
// 0 is default value, which means (_tx_share_memory_limit_percentage = memstore_limit_percentage + 10)
is_valid = true;
@ -149,7 +156,10 @@ bool less_or_equal_tx_share_limit(const uint64_t tenant_id, const int64_t value)
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
if (tenant_config.is_valid()) {
tx_share_limit = tenant_config->_tx_share_memory_limit_percentage;
if (0 == tx_share_limit) {
if (0 == value) {
// 0 is default value, which means memstore limit percentage will adjust itself.
bool_ret = true;
} else if (0 == tx_share_limit) {
// 0 is default value, which means (_tx_share_memory_limit_percentage = memstore_limit_percentage + 10)
bool_ret = true;
} else if (value > 0 && value < 100 && value <= tx_share_limit) {

View File

@ -119,6 +119,7 @@ public:
private:
DISALLOW_COPY_AND_ASSIGN(ObConfigMemstoreLimitChecker);
};
class ObConfigTxDataLimitChecker
{
public:

View File

@ -56,6 +56,7 @@ const char* const FREEZE_TRIGGER_PERCENTAGE = "freeze_trigger_percentage";
const char* const WRITING_THROTTLEIUNG_TRIGGER_PERCENTAGE = "writing_throttling_trigger_percentage";
const char* const _TX_SHARE_MEMORY_LIMIT_PERCENTAGE = "_tx_share_memory_limit_percentage";
const char* const MEMSTORE_LIMIT_PERCENTAGE = "memstore_limit_percentage";
const char* const TENANT_MEMSTORE_LIMIT_PERCENTAGE = "_memstore_limit_percentage";
const char* const _TX_DATA_MEMORY_LIMIT_PERCENTAGE = "_tx_data_memory_limit_percentage";
const char* const _MDS_MEMORY_LIMIT_PERCENTAGE = "_mds_memory_limit_percentage";
const char* const COMPATIBLE = "compatible";

View File

@ -369,10 +369,26 @@ DEF_BOOL(_nested_loop_join_enabled, OB_TENANT_PARAMETER, "True",
ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
// tenant memtable consumption related
DEF_INT(memstore_limit_percentage, OB_TENANT_PARAMETER, "50", "(0, 100)",
DEF_INT(memstore_limit_percentage, OB_CLUSTER_PARAMETER, "0", "[0, 100)",
"used in calculating the value of MEMSTORE_LIMIT parameter: "
"memstore_limit_percentage = memstore_limit / memory_size,memory_size, "
"where MEMORY_SIZE is determined when the tenant is created. Range: (0, 100)",
"memstore_limit_percentage = memstore_limit / memory_size, "
"where MEMORY_SIZE is determined when the tenant is created. Range: [0, 100). "
"1. the system will use memstore_limit_percentage if only memstore_limit_percentage is set."
"2. the system will use _memstore_limit_percentage if both memstore_limit_percentage and "
"_memstore_limit_percentage is set."
"3. the system will adjust automatically if both memstore_limit_percentage and "
"_memstore_limit_percentage set to 0(by default).",
ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_INT(_memstore_limit_percentage, OB_TENANT_PARAMETER, "0", "[0, 100)",
"used in calculating the value of MEMSTORE_LIMIT parameter: "
"_memstore_limit_percentage = memstore_limit / memory_size, "
"where MEMORY_SIZE is determined when the tenant is created. Range: [0, 100). "
"1. the system will use memstore_limit_percentage if only memstore_limit_percentage is set."
"2. the system will use _memstore_limit_percentage if both memstore_limit_percentage and "
"_memstore_limit_percentage is set."
"3. the system will adjust automatically if both memstore_limit_percentage and "
"_memstore_limit_percentage set to 0(by default).",
ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_INT(freeze_trigger_percentage, OB_TENANT_PARAMETER, "20", "(0, 100)",
"the threshold of the size of the mem store when freeze will be triggered. Rang:(0,100)",

View File

@ -14,6 +14,7 @@
#include "observer/omt/ob_tenant_config_mgr.h"
#include "lib/alloc/alloc_func.h"
#include "storage/tx_storage/ob_tenant_freezer.h"
namespace oceanbase {
@ -41,7 +42,7 @@ void FakeAllocatorForTxShare::init_throttle_config(int64_t &resource_limit,
int64_t share_mem_limit = tenant_config->_tx_share_memory_limit_percentage;
// if _tx_share_memory_limit_percentage equals 1, use (memstore_limit_percentage + 10) as default value
if (0 == share_mem_limit) {
share_mem_limit = tenant_config->memstore_limit_percentage + 10;
share_mem_limit = MTL(ObTenantFreezer*)->get_memstore_limit_percentage() + 10;
}
resource_limit = total_memory * share_mem_limit / 100LL;
trigger_percentage = tenant_config->writing_throttling_trigger_percentage;

View File

@ -19,6 +19,7 @@
#include "src/sql/dtl/ob_dtl_tenant_mem_manager.h"
#include "share/ob_occam_time_guard.h"
#include "lib/utility/ob_tracepoint.h"
#include "storage/tx_storage/ob_tenant_freezer.h"
using namespace oceanbase::common;
using namespace oceanbase::lib;
@ -68,12 +69,8 @@ int ObDtlChannelMemManager::get_max_mem_percent()
int ObDtlChannelMemManager::get_memstore_limit_percentage_()
{
int ret = OB_SUCCESS;
ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_));
if (tenant_config.is_valid()) {
memstore_limit_percent_ = tenant_config->memstore_limit_percentage;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("failed to init tenant config", K(tenant_id_), K(ret));
MTL_SWITCH(tenant_id_) {
memstore_limit_percent_ = MTL(ObTenantFreezer*)->get_memstore_limit_percentage();
}
return ret;
}

View File

@ -1004,6 +1004,11 @@ int ObTenantFreezer::get_tenant_memstore_limit(int64_t &mem_limit)
return ret;
}
int64_t ObTenantFreezer::get_memstore_limit_percentage()
{
return get_memstore_limit_percentage_();
}
int ObTenantFreezer::get_tenant_mem_usage_(ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
@ -1242,19 +1247,36 @@ int64_t ObTenantFreezer::get_freeze_trigger_percentage_()
int64_t ObTenantFreezer::get_memstore_limit_percentage_()
{
int ret = OB_SUCCESS;
static const int64_t DEFAULT_MEMSTORE_LIMIT_PERCENTAGE = 50;
int64_t percent = DEFAULT_MEMSTORE_LIMIT_PERCENTAGE;
static const int64_t SMALL_TENANT_MEMORY_LIMIT = 8 * 1024 * 1024 * 1024L; // 8G
static const int64_t SMALL_MEMSTORE_LIMIT_PERCENTAGE = 40;
static const int64_t LARGE_MEMSTORE_LIMIT_PERCENTAGE = 50;
const int64_t tenant_memory = lib::get_tenant_memory_limit(MTL_ID());
const int64_t cluster_memstore_limit_percent = GCONF.memstore_limit_percentage;
int64_t tenant_memstore_limit_percent = 0;
int64_t percent = 0;
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
percent = tenant_config->memstore_limit_percentage;
tenant_memstore_limit_percent = tenant_config->_memstore_limit_percentage;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("memstore limit percentage is invalid", K(ret));
}
if (tenant_memstore_limit_percent != 0) {
percent = tenant_memstore_limit_percent;
} else if (cluster_memstore_limit_percent != 0) {
percent = cluster_memstore_limit_percent;
} else {
// both is default value, adjust automatically
if (tenant_memory <= SMALL_TENANT_MEMORY_LIMIT) {
percent = SMALL_MEMSTORE_LIMIT_PERCENTAGE;
} else {
percent = LARGE_MEMSTORE_LIMIT_PERCENTAGE;
}
}
return percent;
}
int ObTenantFreezer::post_freeze_request_(
const storage::ObFreezeType freeze_type,
const int64_t try_frozen_scn)

View File

@ -118,6 +118,8 @@ public:
const bool force_refresh = true);
// get the tenant memstore limit.
int get_tenant_memstore_limit(int64_t &mem_limit);
// get the memstore limit percentage
static int64_t get_memstore_limit_percentage();
// this is used to check if the tenant's memstore is out at user side.
int check_memstore_full(bool &is_out_of_mem);
// this is used for internal check rather than user side.

View File

@ -142,12 +142,6 @@ int ObTenantInfo::update_frozen_scn(int64_t frozen_scn)
return ret;
}
int64_t ObTenantInfo::mem_memstore_left() const
{
uint64_t memstore_hold = get_tenant_memory_hold(tenant_id_, ObCtxIds::MEMSTORE_CTX_ID);
return max(0, mem_memstore_limit_ - (int64_t)memstore_hold);
}
void ObTenantInfo::get_mem_limit(int64_t &lower_limit, int64_t &upper_limit) const
{
SpinRLockGuard guard(lock_);
@ -163,11 +157,11 @@ void ObTenantInfo::update_mem_limit(const int64_t lower_limit,
mem_upper_limit_ = upper_limit;
}
void ObTenantInfo::update_memstore_limit(const int64_t memstore_limit_percentage)
void ObTenantInfo::update_memstore_limit(const int64_t tenant_memstore_limit_percentage)
{
SpinWLockGuard guard(lock_);
int64_t tmp_var = mem_upper_limit_ / 100;
mem_memstore_limit_ = tmp_var * memstore_limit_percentage;
mem_memstore_limit_ = tmp_var * tenant_memstore_limit_percentage;
}
int64_t ObTenantInfo::get_memstore_limit() const

View File

@ -114,11 +114,10 @@ public:
virtual ~ObTenantInfo() { reset(); }
void reset();
int update_frozen_scn(int64_t frozen_scn);
int64_t mem_memstore_left() const;
void update_mem_limit(const int64_t lower_limit, const int64_t upper_limit);
void get_mem_limit(int64_t &lower_limit, int64_t &upper_limit) const;
void update_memstore_limit(const int64_t memstore_limit_percentage);
void update_memstore_limit(const int64_t tenant_memstore_limit_percentage);
int64_t get_memstore_limit() const;
bool is_memstore_limit_changed(const int64_t curr_memstore_limit_percentage) const;
void get_freeze_ctx(ObTenantFreezeCtx &ctx) const;

View File

@ -342,6 +342,7 @@ _max_schema_slot_num
_max_tablet_cnt_per_gb
_mds_memory_limit_percentage
_memory_large_chunk_cache_size
_memstore_limit_percentage
_migrate_block_verify_level
_minor_compaction_amplification_factor
_min_malloc_sample_interval