hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

BUGFIX: break deadlock at TenantFreezer

Browse Source
This commit is contained in:
obdev
2022-11-08 03:42:35 +00:00
committed by wangzelin.wzl
parent 4e2bd1d9a5
commit 92233f2269
4 changed files with 259 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

349
src/storage/tx_storage/ob_tenant_freezer.cpp
View File

@ -204,7 +204,7 @@ int ObTenantFreezer::ls_freeze_(ObLS *ls)
return ret;
}
int ObTenantFreezer::tenant_freeze()
int ObTenantFreezer::tenant_freeze_()
{
int ret = OB_SUCCESS;
int first_fail_ret = OB_SUCCESS;
@ -212,10 +212,7 @@ int ObTenantFreezer::tenant_freeze()
ObLSService *ls_srv = MTL(ObLSService *);
FLOG_INFO("[TenantFreezer] tenant_freeze start", KR(ret));
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret));
} else if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) {
if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) {
LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret));
} else {
ObLS *ls = nullptr;
@ -349,48 +346,34 @@ int ObTenantFreezer::get_tenant_tx_data_mem_used_(int64_t &tenant_tx_data_mem_us
return ret;
}
int ObTenantFreezer::check_and_freeze_normal_data_()
int ObTenantFreezer::check_and_freeze_normal_data_(ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
bool upgrade_mode = GCONF.in_major_version_upgrade_mode();
int tmp_ret = OB_SUCCESS;
bool need_freeze = false;
int64_t active_memstore_used = 0;
int64_t total_memstore_used = 0;
int64_t total_memstore_hold = 0;
int64_t memstore_freeze_trigger = 0;
if (OB_UNLIKELY(upgrade_mode)) {
// skip trigger freeze while upgrading
} else {
{
SpinRLockGuard guard(lock_);
if (!tenant_info_.is_loaded_) {
// do nothing
} else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) {
LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret));
} else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used,
total_memstore_used,
total_memstore_hold))) {
LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret));
} else {
need_freeze = need_freeze_(active_memstore_used,
memstore_freeze_trigger);
if (need_freeze && !is_minor_need_slow_(total_memstore_hold, memstore_freeze_trigger)) {
unset_tenant_slow_freeze_();
}
log_frozen_memstore_info_if_need_(active_memstore_used, total_memstore_used,
total_memstore_hold, memstore_freeze_trigger);
halt_prewarm_if_need_(memstore_freeze_trigger, total_memstore_hold);
if (OB_FAIL(get_freeze_trigger_(ctx))) {
LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret));
} else if (OB_FAIL(get_tenant_mem_usage_(ctx))) {
LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret));
} else {
need_freeze = need_freeze_(ctx);
if (need_freeze && !is_minor_need_slow_(ctx)) {
unset_tenant_slow_freeze_();
}
log_frozen_memstore_info_if_need_(ctx);
halt_prewarm_if_need_(ctx);
}
// must out of the lock, to make sure there is no deadlock, just because of tenant freeze hung.
if (OB_TMP_FAIL(do_major_if_need_(need_freeze))) {
LOG_WARN("[TenantFreezer] fail to do major freeze", K(tmp_ret));
}
if (need_freeze) {
if (OB_TMP_FAIL(do_minor_freeze_(active_memstore_used,
memstore_freeze_trigger))) {
if (OB_TMP_FAIL(do_minor_freeze_(ctx))) {
LOG_WARN("[TenantFreezer] fail to do minor freeze", K(tmp_ret));
}
}
@ -444,11 +427,15 @@ int ObTenantFreezer::check_and_do_freeze()
int ret = OB_SUCCESS;
int64_t check_and_freeze_start_ts = ObTimeUtil::current_time();
ObTenantFreezeCtx ctx;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else if (OB_FAIL(check_and_freeze_normal_data_())) {
} else if (!tenant_info_.is_loaded_) {
// do nothing
} else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) {
} else if (OB_FAIL(check_and_freeze_normal_data_(ctx))) {
LOG_WARN("[TenantFreezer] check and freeze normal data failed.", KR(ret));
} else if (OB_FAIL(check_and_freeze_tx_data_())) {
LOG_WARN("[TenantFreezer] check and freeze tx data failed.", KR(ret));
@ -477,27 +464,25 @@ int ObTenantFreezer::retry_failed_major_freeze_(bool &triggered)
return ret;
}
int ObTenantFreezer::set_tenant_freezing()
int ObTenantFreezer::set_tenant_freezing_()
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
SpinRLockGuard guard(lock_);
ATOMIC_AAF(&tenant_info_.freeze_cnt_, 1);
}
return ret;
}
int ObTenantFreezer::unset_tenant_freezing(const bool rollback_freeze_cnt)
int ObTenantFreezer::unset_tenant_freezing_(const bool rollback_freeze_cnt)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
SpinRLockGuard guard(lock_);
if (rollback_freeze_cnt) {
if (ATOMIC_AAF(&tenant_info_.freeze_cnt_, -1) < 0) {
tenant_info_.freeze_cnt_ = 0;
@ -517,7 +502,6 @@ int ObTenantFreezer::set_tenant_slow_freeze(
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
SpinRLockGuard guard(lock_);
if (!tenant_info_.slow_freeze_) {
bool success = ATOMIC_BCAS(&tenant_info_.slow_freeze_, false, true);
if (success) {
@ -561,7 +545,6 @@ int ObTenantFreezer::unset_tenant_slow_freeze()
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
SpinRLockGuard guard(lock_);
ret = unset_tenant_slow_freeze_();
}
return ret;
@ -575,7 +558,6 @@ int ObTenantFreezer::unset_tenant_slow_freeze(const common::ObTabletID &tablet_i
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
SpinRLockGuard guard(lock_);
if (tenant_info_.slow_freeze_ && tenant_info_.slow_tablet_ == tablet_id) {
bool success = ATOMIC_BCAS(&tenant_info_.slow_freeze_, true, false);
if (success) {
@ -619,25 +601,25 @@ int ObTenantFreezer::set_tenant_mem_limit(
KR(ret));
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
SpinWLockGuard guard(lock_); // It should be possible to change to a read lock here, this lock is a structural lock, it is not appropriate to borrow
int64_t memstore_freeze_trigger_limit = 0;
tenant_info_.mem_lower_limit_ = lower_limit;
tenant_info_.mem_upper_limit_ = upper_limit;
ObTenantFreezeCtx ctx;
tenant_info_.update_mem_limit(lower_limit, upper_limit);
if (NULL != config_) {
int64_t tmp_var = upper_limit / 100;
tenant_info_.mem_memstore_limit_ = tmp_var * config_->memstore_limit_percentage;
if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger_limit))) {
tenant_info_.update_memstore_limit(config_->memstore_limit_percentage);
}
tenant_info_.is_loaded_ = true;
tenant_info_.get_freeze_ctx(ctx);
if (NULL != config_) {
if (OB_FAIL(get_freeze_trigger_(ctx))) {
LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id));
}
}
tenant_info_.is_loaded_ = true;
if (OB_SUCC(ret)) {
LOG_INFO("[TenantFreezer] set tenant mem limit",
"tenant id", tenant_id,
"mem_lower_limit", lower_limit,
"mem_upper_limit", upper_limit,
"mem_memstore_limit", tenant_info_.mem_memstore_limit_,
"memstore_freeze_trigger_limit", memstore_freeze_trigger_limit,
"mem_memstore_limit", ctx.mem_memstore_limit_,
"memstore_freeze_trigger_limit", ctx.memstore_freeze_trigger_,
"mem_tenant_limit", get_tenant_memory_limit(tenant_info_.tenant_id_),
"mem_tenant_hold", get_tenant_memory_hold(tenant_info_.tenant_id_),
"mem_memstore_used", get_tenant_memory_hold(tenant_info_.tenant_id_,
@ -660,12 +642,10 @@ int ObTenantFreezer::get_tenant_mem_limit(
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
SpinRLockGuard guard(lock_);
if (false == tenant_info_.is_loaded_) {
ret = OB_NOT_REGISTERED;
} else {
lower_limit = tenant_info_.mem_lower_limit_;
upper_limit = tenant_info_.mem_upper_limit_;
tenant_info_.get_mem_limit(lower_limit, upper_limit);
}
}
return ret;
@ -689,6 +669,7 @@ int ObTenantFreezer::get_tenant_memstore_cond(
RLOCAL(int64_t, last_memstore_freeze_trigger);
RLOCAL(int64_t, last_memstore_limit);
RLOCAL(int64_t, last_freeze_cnt);
ObTenantFreezeCtx ctx;
active_memstore_used = 0;
total_memstore_used = 0;
@ -706,19 +687,20 @@ int ObTenantFreezer::get_tenant_memstore_cond(
memstore_limit = last_memstore_limit;
freeze_cnt = last_freeze_cnt;
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
SpinRLockGuard guard(lock_);
const uint64_t tenant_id = MTL_ID();
if (false == tenant_info_.is_loaded_) {
ret = OB_ENTRY_NOT_EXIST;
LOG_INFO("[TenantFreezer] This tenant not exist", K(tenant_id), KR(ret));
} else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used,
total_memstore_used,
unused))) {
} else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) {
} else if (OB_FAIL(get_tenant_mem_usage_(ctx))) {
LOG_WARN("[TenantFreezer] failed to get tenant mem usage", KR(ret), K(tenant_id));
} else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) {
LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id));
} else if (OB_FAIL(get_freeze_trigger_(ctx))) {
LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id));
} else {
memstore_limit = tenant_info_.mem_memstore_limit_;
memstore_limit = ctx.mem_memstore_limit_;
active_memstore_used = ctx.active_memstore_used_;
total_memstore_used = ctx.total_memstore_used_;
memstore_freeze_trigger = ctx.memstore_freeze_trigger_;
freeze_cnt = tenant_info_.freeze_cnt_;
// cache the result
@ -742,56 +724,44 @@ int ObTenantFreezer::get_tenant_memstore_limit(int64_t &mem_limit)
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
SpinRLockGuard guard(lock_);
if (false == tenant_info_.is_loaded_) {
mem_limit = INT64_MAX;
LOG_INFO("[TenantFreezer] This tenant not exist", K(tenant_id), KR(ret));
} else {
mem_limit = tenant_info_.mem_memstore_limit_;
mem_limit = tenant_info_.get_memstore_limit();
}
}
return ret;
}
int ObTenantFreezer::get_tenant_mem_usage_(
int64_t &active_memstore_used,
int64_t &total_memstore_used,
int64_t &total_memstore_hold)
int ObTenantFreezer::get_tenant_mem_usage_(ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
ObTenantMemstoreAllocator *tenant_allocator = NULL;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
int64_t active_memstore_used = 0;
int64_t total_memstore_used = 0;
int64_t total_memstore_hold = 0;
const uint64_t tenant_id = MTL_ID();
if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id,
tenant_allocator))) {
LOG_WARN("[TenantFreezer] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id));
} else if (NULL == tenant_allocator) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("[TenantFreezer] tenant memstore allocator is NULL", KR(ret), K(tenant_id));
} else {
const uint64_t tenant_id = tenant_info_.tenant_id_;
if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id,
tenant_allocator))) {
LOG_WARN("[TenantFreezer] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id));
} else if (NULL == tenant_allocator) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("[TenantFreezer] tenant memstore allocator is NULL", KR(ret), K(tenant_id));
} else {
active_memstore_used = tenant_allocator->get_mem_active_memstore_used();
total_memstore_used = tenant_allocator->get_mem_total_memstore_used();
total_memstore_hold = get_tenant_memory_hold(tenant_id,
ObCtxIds::MEMSTORE_CTX_ID);
}
active_memstore_used = tenant_allocator->get_mem_active_memstore_used();
total_memstore_used = tenant_allocator->get_mem_total_memstore_used();
total_memstore_hold = get_tenant_memory_hold(tenant_id,
ObCtxIds::MEMSTORE_CTX_ID);
}
ctx.active_memstore_used_ = active_memstore_used;
ctx.total_memstore_used_ = total_memstore_used;
ctx.total_memstore_hold_ = total_memstore_hold;
return ret;
}
int ObTenantFreezer::get_freeze_trigger_(int64_t &memstore_freeze_trigger)
{
int64_t not_used = 0;
int64_t not_used2 = 0;
return get_freeze_trigger_(not_used,
not_used2,
memstore_freeze_trigger);
}
static inline bool is_add_overflow(int64_t first, int64_t second, int64_t &res)
{
if (first + second < 0) {
@ -802,41 +772,18 @@ static inline bool is_add_overflow(int64_t first, int64_t second, int64_t &res)
}
}
int ObTenantFreezer::get_mem_remain_trigger_(
int64_t &mem_remain_trigger)
{
int ret = OB_SUCCESS;
const int64_t tenant_id = tenant_info_.tenant_id_;
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
double memstore_limit = tenant_info_.mem_memstore_limit_;
// 1. trigger by write throttling
if (!tenant_config.is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("[TenantFreezer] failed to get tenant config", KR(ret));
} else {
int64_t trigger_percentage = tenant_config->writing_throttling_trigger_percentage;
mem_remain_trigger = memstore_limit * (100 - trigger_percentage) / 100 / 0.95;
}
return ret;
}
int ObTenantFreezer::get_freeze_trigger_(
/* Now the maximum memory size that the memstore module can preempt and obtain */
int64_t &max_mem_memstore_can_get_now,
int64_t &kv_cache_mem,
int64_t &memstore_freeze_trigger)
int ObTenantFreezer::get_freeze_trigger_(ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
ObTenantResourceMgrHandle resource_handle;
const uint64_t tenant_id = tenant_info_.tenant_id_;
const int64_t mem_memstore_limit = tenant_info_.mem_memstore_limit_;
if (OB_UNLIKELY(NULL == config_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("[TenantFreezer] config_ is nullptr", KR(ret), K(tenant_id));
} else if (OB_FAIL(ObResourceMgr::get_instance().
get_tenant_resource_mgr(tenant_id,
resource_handle))) {
const uint64_t tenant_id = MTL_ID();
const int64_t mem_memstore_limit = ctx.mem_memstore_limit_;
int64_t kv_cache_mem = 0;
int64_t memstore_freeze_trigger = 0;
int64_t max_mem_memstore_can_get_now = 0;
if (OB_FAIL(ObResourceMgr::get_instance().
get_tenant_resource_mgr(tenant_id,
resource_handle))) {
LOG_WARN("[TenantFreezer] fail to get resource mgr", KR(ret), K(tenant_id));
ret = OB_SUCCESS;
memstore_freeze_trigger =
@ -881,6 +828,10 @@ int ObTenantFreezer::get_freeze_trigger_(
memstore_freeze_trigger = min / 100 * get_freeze_trigger_percentage_();
}
}
// result
ctx.max_mem_memstore_can_get_now_ = max_mem_memstore_can_get_now;
ctx.memstore_freeze_trigger_ = memstore_freeze_trigger;
ctx.kvcache_mem_ = kv_cache_mem;
return ret;
}
@ -892,6 +843,7 @@ int ObTenantFreezer::check_tenant_out_of_memstore_limit(bool &is_out_of_mem)
RLOCAL(int64_t, last_check_timestamp);
RLOCAL(bool, last_result);
int64_t current_time = OB_TSC_TIMESTAMP.current_time();
ObTenantFreezeCtx ctx;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
@ -902,19 +854,14 @@ int ObTenantFreezer::check_tenant_out_of_memstore_limit(bool &is_out_of_mem)
// Check once when the last memory burst or tenant_id does not match or the interval reaches the threshold
is_out_of_mem = false;
} else {
int64_t active_memstore_used = 0;
int64_t total_memstore_used = 0;
int64_t total_memstore_hold = 0;
SpinRLockGuard guard(lock_);
if (false == tenant_info_.is_loaded_) {
is_out_of_mem = false;
LOG_INFO("[TenantFreezer] This tenant not exist", K(tenant_id), KR(ret));
} else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used,
total_memstore_used,
total_memstore_hold))) {
} else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) {
} else if (OB_FAIL(get_tenant_mem_usage_(ctx))) {
LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret), K(tenant_info_.tenant_id_));
} else {
is_out_of_mem = (total_memstore_hold > tenant_info_.mem_memstore_limit_);
is_out_of_mem = (ctx.total_memstore_hold_ > ctx.mem_memstore_limit_);
}
last_check_timestamp = current_time;
}
@ -930,32 +877,26 @@ bool ObTenantFreezer::tenant_need_major_freeze()
{
int ret = OB_SUCCESS;
bool bool_ret = false;
int64_t active_memstore_used = 0;
int64_t total_memstore_used = 0;
int64_t total_memstore_hold = 0;
int64_t memstore_freeze_trigger = 0;
ObTenantFreezeCtx ctx;
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("tenant manager not init", K(ret));
} else {
SpinRLockGuard guard(lock_);
if (!tenant_info_.is_loaded_) {
// do nothing
} else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) {
} else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) {
} else if (OB_FAIL(get_freeze_trigger_(ctx))) {
LOG_WARN("fail to get minor freeze trigger", K(ret), K(tenant_info_.tenant_id_));
} else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used,
total_memstore_used,
total_memstore_hold))) {
} else if (OB_FAIL(get_tenant_mem_usage_(ctx))) {
LOG_WARN("fail to get mem usage", K(ret), K(tenant_info_.tenant_id_));
} else {
bool_ret = need_freeze_(active_memstore_used,
memstore_freeze_trigger);
bool_ret = need_freeze_(ctx);
if (bool_ret) {
LOG_INFO("A major freeze is needed",
"active_memstore_used_",
active_memstore_used,
ctx.active_memstore_used_,
"memstore_freeze_trigger_limit_",
memstore_freeze_trigger,
ctx.memstore_freeze_trigger_,
"tenant_id",
tenant_info_.tenant_id_);
}
@ -964,7 +905,7 @@ bool ObTenantFreezer::tenant_need_major_freeze()
return bool_ret;
}
int64_t ObTenantFreezer::get_freeze_trigger_percentage_() const
int64_t ObTenantFreezer::get_freeze_trigger_percentage_()
{
static const int64_t DEFAULT_FREEZE_TRIGGER_PERCENTAGE = 20;
int64_t percent = DEFAULT_FREEZE_TRIGGER_PERCENTAGE;
@ -1041,11 +982,8 @@ void ObTenantFreezer::reload_config()
freeze_trigger_percentage,
KR(ret));
} else {
SpinWLockGuard guard(lock_); // It should be possible to change to a read lock here, this lock is a structural lock, it is not appropriate to borrow
if (true == tenant_info_.is_loaded_) {
int64_t tmp_var = tenant_info_.mem_upper_limit_ / 100;
tenant_info_.mem_memstore_limit_ =
tmp_var * config_->memstore_limit_percentage;
tenant_info_.update_memstore_limit(config_->memstore_limit_percentage);
}
}
if (OB_SUCCESS == ret) {
@ -1063,25 +1001,16 @@ int ObTenantFreezer::print_tenant_usage(
int64_t &pos)
{
int ret = OB_SUCCESS;
ObTenantFreezeCtx ctx;
lib::ObMallocAllocator *mallocator = lib::ObMallocAllocator::get_instance();
int64_t active_memstore_used = 0;
int64_t total_memstore_used = 0;
int64_t total_memstore_hold = 0;
int64_t memstore_freeze_trigger_limit = 0;
int64_t max_mem_memstore_can_get_now = 0;
int64_t kv_cache_mem = 0;
SpinWLockGuard guard(lock_);
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret));
} else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used,
total_memstore_used,
total_memstore_hold))) {
} else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) {
} else if (OB_FAIL(get_tenant_mem_usage_(ctx))) {
LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret), K(tenant_info_.tenant_id_));
} else if (OB_FAIL(get_freeze_trigger_(max_mem_memstore_can_get_now,
kv_cache_mem,
memstore_freeze_trigger_limit))) {
} else if (OB_FAIL(get_freeze_trigger_(ctx))) {
LOG_WARN("[TenantFreezer] get tenant minor freeze trigger error", KR(ret), K(tenant_info_.tenant_id_));
} else {
ret = databuff_printf(print_buf, buf_len, pos,
@ -1094,21 +1023,18 @@ int ObTenantFreezer::print_tenant_usage(
"memstore_limit=% '15ld "
"mem_tenant_limit=% '15ld "
"mem_tenant_hold=% '15ld "
"mem_memstore_used=% '15ld "
"kv_cache_mem=% '15ld "
"max_mem_memstore_can_get_now=% '15ld\n",
tenant_info_.tenant_id_,
active_memstore_used,
total_memstore_used,
total_memstore_hold,
memstore_freeze_trigger_limit,
tenant_info_.mem_memstore_limit_,
ctx.active_memstore_used_,
ctx.total_memstore_used_,
ctx.total_memstore_hold_,
ctx.memstore_freeze_trigger_,
ctx.mem_memstore_limit_,
get_tenant_memory_limit(tenant_info_.tenant_id_),
get_tenant_memory_hold(tenant_info_.tenant_id_),
get_tenant_memory_hold(tenant_info_.tenant_id_,
ObCtxIds::MEMSTORE_CTX_ID),
kv_cache_mem,
max_mem_memstore_can_get_now);
ctx.kvcache_mem_,
ctx.max_mem_memstore_can_get_now_);
}
if (!OB_ISNULL(mallocator)) {
@ -1134,17 +1060,14 @@ int ObTenantFreezer::get_global_frozen_scn_(int64_t &frozen_scn)
return ret;
}
bool ObTenantFreezer::need_freeze_(
const int64_t active_memstore_used,
const int64_t memstore_freeze_trigger)
bool ObTenantFreezer::need_freeze_(const ObTenantFreezeCtx &ctx)
{
bool need_freeze = false;
const int64_t tenant_id = tenant_info_.tenant_id_;
// 1. trigger by active memstore used.
if (active_memstore_used > memstore_freeze_trigger) {
if (ctx.active_memstore_used_ > ctx.memstore_freeze_trigger_) {
need_freeze = true;
LOG_INFO("[TenantFreezer] A minor freeze is needed by active memstore used.",
K(active_memstore_used), K(memstore_freeze_trigger), K(tenant_id));
K(ctx.active_memstore_used_), K(ctx.memstore_freeze_trigger_));
}
return need_freeze;
}
@ -1160,16 +1083,14 @@ bool ObTenantFreezer::is_major_freeze_turn_()
return (major_compact_trigger != 0 && freeze_cnt >= major_compact_trigger);
}
bool ObTenantFreezer::is_minor_need_slow_(
const int64_t total_memstore_hold,
const int64_t memstore_freeze_trigger)
bool ObTenantFreezer::is_minor_need_slow_(const ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
bool need_slow = false;
if (tenant_info_.slow_freeze_) {
need_slow = true;
int64_t now = ObTimeUtility::fast_current_time();
if (total_memstore_hold <= memstore_freeze_trigger) {
if (ctx.total_memstore_hold_ <= ctx.memstore_freeze_trigger_) {
// no need minor freeze
} else if (now - tenant_info_.slow_freeze_timestamp_ >= SLOW_FREEZE_INTERVAL) {
need_slow = false;
@ -1180,15 +1101,14 @@ bool ObTenantFreezer::is_minor_need_slow_(
return need_slow;
}
int ObTenantFreezer::do_minor_freeze_(const int64_t active_memstore_used,
const int64_t memstore_freeze_trigger)
int ObTenantFreezer::do_minor_freeze_(const ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
bool rollback_freeze_cnt = false;
LOG_INFO("[TenantFreezer] A minor freeze is needed",
"active_memstore_used_", active_memstore_used,
"memstore_freeze_trigger", memstore_freeze_trigger,
"active_memstore_used_", ctx.active_memstore_used_,
"memstore_freeze_trigger", ctx.memstore_freeze_trigger_,
"mem_tenant_remain", get_tenant_memory_remain(MTL_ID()),
"mem_tenant_limit", get_tenant_memory_limit(MTL_ID()),
"mem_tenant_hold", get_tenant_memory_hold(MTL_ID()),
@ -1196,10 +1116,10 @@ int ObTenantFreezer::do_minor_freeze_(const int64_t active_memstore_used,
ObCtxIds::MEMSTORE_CTX_ID),
"tenant_id", MTL_ID());
if (OB_FAIL(set_tenant_freezing())) {
if (OB_FAIL(set_tenant_freezing_())) {
} else {
bool rollback_freeze_cnt = false;
if (OB_FAIL(tenant_freeze())) {
if (OB_FAIL(tenant_freeze_())) {
rollback_freeze_cnt = true;
LOG_WARN("fail to minor freeze", K(ret));
} else {
@ -1208,7 +1128,7 @@ int ObTenantFreezer::do_minor_freeze_(const int64_t active_memstore_used,
// clear freezing mark for tenant
int tmp_ret = OB_SUCCESS;
if (OB_UNLIKELY(OB_SUCCESS !=
(tmp_ret = unset_tenant_freezing(rollback_freeze_cnt)))) {
(tmp_ret = unset_tenant_freezing_(rollback_freeze_cnt)))) {
LOG_WARN("unset tenant freezing mark failed", K(tmp_ret));
if (OB_SUCC(ret)) {
ret = tmp_ret;
@ -1230,21 +1150,18 @@ int ObTenantFreezer::do_major_if_need_(const bool need_freeze)
if (OB_TMP_FAIL(retry_failed_major_freeze_(major_triggered))) {
LOG_WARN("fail to do major freeze due to previous failure", K(tmp_ret));
}
// update frozen scn
if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) {
if (!tenant_info_.is_loaded_) {
// do nothing
// update frozen scn
} else if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) {
LOG_WARN("fail to get global frozen version", K(ret));
} else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) {
LOG_WARN("fail to update frozen version", K(ret), K(frozen_scn), K_(tenant_info));
} else {
SpinRLockGuard guard(lock_);
if (!tenant_info_.is_loaded_) {
// do nothing
} else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) {
LOG_WARN("fail to update frozen version", K(ret), K(frozen_scn), K_(tenant_info));
} else {
need_major = (need_freeze &&
!major_triggered &&
is_major_freeze_turn_());
curr_frozen_scn = tenant_info_.frozen_scn_;
}
need_major = (need_freeze &&
!major_triggered &&
is_major_freeze_turn_());
curr_frozen_scn = tenant_info_.frozen_scn_;
}
if (need_major) {
if (OB_FAIL(do_major_freeze_(curr_frozen_scn))) {
@ -1268,22 +1185,18 @@ int ObTenantFreezer::do_major_freeze_(const int64_t try_frozen_scn)
return ret;
}
void ObTenantFreezer::log_frozen_memstore_info_if_need_(
const int64_t active_memstore_used,
const int64_t total_memstore_used,
const int64_t total_memstore_hold,
const int64_t memstore_freeze_trigger)
void ObTenantFreezer::log_frozen_memstore_info_if_need_(const ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
ObTenantMemstoreAllocator *tenant_allocator = NULL;
if (total_memstore_hold > memstore_freeze_trigger) {
if (ctx.total_memstore_hold_ > ctx.memstore_freeze_trigger_) {
// There is an unreleased memstable
LOG_INFO("[TenantFreezer] tenant have inactive memstores",
K(active_memstore_used),
K(total_memstore_used),
K(total_memstore_hold),
K(ctx.active_memstore_used_),
K(ctx.total_memstore_used_),
K(ctx.total_memstore_hold_),
"memstore_freeze_trigger_limit_",
memstore_freeze_trigger,
ctx.memstore_freeze_trigger_,
"tenant_id",
MTL_ID());
@ -1299,15 +1212,13 @@ void ObTenantFreezer::log_frozen_memstore_info_if_need_(
}
}
void ObTenantFreezer::halt_prewarm_if_need_(
const int64_t memstore_freeze_trigger,
const int64_t total_memstore_hold)
void ObTenantFreezer::halt_prewarm_if_need_(const ObTenantFreezeCtx &ctx)
{
int ret = OB_SUCCESS;
// When the memory is tight, try to abort the warm-up to release memstore
int64_t mem_danger_limit = tenant_info_.mem_memstore_limit_
- ((tenant_info_.mem_memstore_limit_ - memstore_freeze_trigger) >> 2);
if (total_memstore_hold > mem_danger_limit) {
int64_t mem_danger_limit = ctx.mem_memstore_limit_
- ((ctx.mem_memstore_limit_ - ctx.memstore_freeze_trigger_) >> 2);
if (ctx.total_memstore_hold_ > mem_danger_limit) {
int64_t curr_ts = ObTimeUtility::current_time();
if (curr_ts - tenant_info_.last_halt_ts_ > 10L * 1000L * 1000L) {
if (OB_FAIL(svr_rpc_proxy_->to(self_).

52
src/storage/tx_storage/ob_tenant_freezer.h
View File

@ -52,20 +52,11 @@ public:
int stop();
void wait();
// freeze all the ls of this tenant.
// return the first failed code.
int tenant_freeze();
// freeze a tablet
int tablet_freeze(const common::ObTabletID &tablet_id,
const bool is_force_freeze=false);
// check if this tenant's memstore is out of range, and trigger minor/major freeze.
int check_and_do_freeze();
// we can only deal with freeze one by one.
// set tenant freezing will prevent a new freeze.
int set_tenant_freezing();
// unset tenant freezing flag.
// @param[in] rollback_freeze_cnt, reduce the tenant's freeze count by 1, if true.
int unset_tenant_freezing(const bool rollback_freeze_cnt);
// If the tenant's freeze process is slowed, we will only freeze one time every
// SLOW_FREEZE_INTERVAL.
@ -128,45 +119,40 @@ public:
ObServerConfig *get_config() { return config_; }
bool exist_ls_freezing();
private:
int ls_freeze_(ObLS *ls);
int64_t get_freeze_trigger_percentage_() const;
static int ls_freeze_(ObLS *ls);
// freeze all the ls of this tenant.
// return the first failed code.
static int tenant_freeze_();
// we can only deal with freeze one by one.
// set tenant freezing will prevent a new freeze.
int set_tenant_freezing_();
// unset tenant freezing flag.
// @param[in] rollback_freeze_cnt, reduce the tenant's freeze count by 1, if true.
int unset_tenant_freezing_(const bool rollback_freeze_cnt);
static int64_t get_freeze_trigger_percentage_();
int post_freeze_request_(const storage::ObFreezeType freeze_type,
const int64_t try_frozen_version);
int retry_failed_major_freeze_(bool &triggered);
int get_global_frozen_scn_(int64_t &frozen_version);
int post_tx_data_freeze_request_();
int get_tenant_mem_usage_(int64_t &active_memstore_used,
int64_t &total_memstore_used,
int64_t &total_memstore_hold);
int get_freeze_trigger_(int64_t &memstore_freeze_trigger);
int get_freeze_trigger_(int64_t &max_mem_memstore_can_get_now,
int64_t &kvcache_mem,
int64_t &memstore_freeze_trigger);
int get_mem_remain_trigger_(int64_t &mem_remain_trigger);
bool need_freeze_(const int64_t active_memstore_used,
const int64_t memstore_freeze_trigger);
bool is_minor_need_slow_(const int64_t mem_total_memstore_hold,
const int64_t memstore_freeze_trigger);
int get_tenant_mem_usage_(ObTenantFreezeCtx &ctx);
static int get_freeze_trigger_(ObTenantFreezeCtx &ctx);
static bool need_freeze_(const ObTenantFreezeCtx &ctx);
bool is_minor_need_slow_(const ObTenantFreezeCtx &ctx);
bool is_major_freeze_turn_();
int do_major_if_need_(const bool need_freeze);
int do_minor_freeze_(const int64_t active_memstore_used,
const int64_t memstore_freeze_trigger);
int do_minor_freeze_(const ObTenantFreezeCtx &ctx);
int do_major_freeze_(const int64_t try_frozen_scn);
void log_frozen_memstore_info_if_need_(const int64_t active_memstore_used,
const int64_t mem_total_memstore_used,
const int64_t mem_total_memstore_hold,
const int64_t memstore_freeze_trigger);
void halt_prewarm_if_need_(const int64_t memstore_freeze_trigger,
const int64_t mem_total_memstore_hold);
void log_frozen_memstore_info_if_need_(const ObTenantFreezeCtx &ctx);
void halt_prewarm_if_need_(const ObTenantFreezeCtx &ctx);
int unset_tenant_slow_freeze_();
int check_and_freeze_normal_data_();
int check_and_freeze_normal_data_(ObTenantFreezeCtx &ctx);
int check_and_freeze_tx_data_();
int get_tenant_tx_data_mem_used_(int64_t &tenant_tx_data_mem_used);
int get_ls_tx_data_mem_used_(ObLS *ls, int64_t &ls_tx_data_mem_used);
private:
bool is_inited_;
bool is_freezing_tx_data_;
SpinRWLock lock_;
ObTenantInfo tenant_info_; // store the mem limit, memstore limit and etc.
obrpc::ObTenantFreezerRpcProxy rpc_proxy_; // used to trigger minor/major freeze
obrpc::ObTenantFreezerRpcCb tenant_mgr_cb_; // callback after the trigger rpc finish.

76
src/storage/tx_storage/ob_tenant_freezer_common.cpp
View File

@ -30,11 +30,34 @@ OB_SERIALIZE_MEMBER(ObTenantFreezeArg,
freeze_type_,
try_frozen_scn_);
ObTenantInfo::ObTenantInfo()
: tenant_id_(INT64_MAX),
mem_lower_limit_(0),
ObTenantFreezeCtx::ObTenantFreezeCtx()
: mem_lower_limit_(0),
mem_upper_limit_(0),
mem_memstore_limit_(0),
memstore_freeze_trigger_(0),
max_mem_memstore_can_get_now_(0),
kvcache_mem_(0),
active_memstore_used_(0),
total_memstore_used_(0),
total_memstore_hold_(0)
{
}
void ObTenantFreezeCtx::reset()
{
mem_lower_limit_ = 0;
mem_upper_limit_ = 0;
mem_memstore_limit_ = 0;
memstore_freeze_trigger_ = 0;
max_mem_memstore_can_get_now_ = 0;
kvcache_mem_ = 0;
active_memstore_used_ = 0;
total_memstore_used_ = 0;
total_memstore_hold_ = 0;
}
ObTenantInfo::ObTenantInfo()
: tenant_id_(INT64_MAX),
is_loaded_(false),
is_freezing_(false),
last_freeze_clock_(0),
@ -43,16 +66,16 @@ ObTenantInfo::ObTenantInfo()
last_halt_ts_(0),
slow_freeze_(false),
slow_freeze_timestamp_(0),
slow_freeze_min_protect_clock_(INT64_MAX)
slow_freeze_min_protect_clock_(INT64_MAX),
mem_lower_limit_(0),
mem_upper_limit_(0),
mem_memstore_limit_(0)
{
}
void ObTenantInfo::reset()
{
tenant_id_ = OB_INVALID_TENANT_ID; // i64 max as invalid.
mem_memstore_limit_ = 0;
mem_lower_limit_ = 0;
mem_upper_limit_ = 0;
is_loaded_ = false;
is_freezing_ = false;
frozen_scn_ = 0;
@ -62,6 +85,9 @@ void ObTenantInfo::reset()
slow_freeze_timestamp_ = 0;
slow_freeze_min_protect_clock_ = INT64_MAX;
slow_tablet_.reset();
mem_memstore_limit_ = 0;
mem_lower_limit_ = 0;
mem_upper_limit_ = 0;
}
int ObTenantInfo::update_frozen_scn(int64_t frozen_scn)
@ -82,5 +108,41 @@ int64_t ObTenantInfo::mem_memstore_left() const
return max(0, mem_memstore_limit_ - (int64_t)memstore_hold);
}
void ObTenantInfo::get_mem_limit(int64_t &lower_limit, int64_t &upper_limit) const
{
SpinRLockGuard guard(lock_);
lower_limit = mem_lower_limit_;
upper_limit = mem_upper_limit_;
}
void ObTenantInfo::update_mem_limit(const int64_t lower_limit,
const int64_t upper_limit)
{
SpinWLockGuard guard(lock_);
mem_lower_limit_ = lower_limit;
mem_upper_limit_ = upper_limit;
}
void ObTenantInfo::update_memstore_limit(const int64_t memstore_limit_percentage)
{
SpinWLockGuard guard(lock_);
int64_t tmp_var = mem_upper_limit_ / 100;
mem_memstore_limit_ = tmp_var * memstore_limit_percentage;
}
int64_t ObTenantInfo::get_memstore_limit() const
{
SpinRLockGuard guard(lock_);
return mem_memstore_limit_;
}
void ObTenantInfo::get_freeze_ctx(ObTenantFreezeCtx &ctx) const
{
SpinRLockGuard guard(lock_);
ctx.mem_lower_limit_ = mem_lower_limit_;
ctx.mem_upper_limit_ = mem_upper_limit_;
ctx.mem_memstore_limit_ = mem_memstore_limit_;
}
} // storage
} // oceanbase

46
src/storage/tx_storage/ob_tenant_freezer_common.h
View File

@ -49,6 +49,32 @@ struct ObRetryMajorInfo
TO_STRING_KV(K_(tenant_id), K_(frozen_scn));
};
// store a snapshot of the tenant info to make sure
// the tenant_info will be a atomic value
struct ObTenantFreezeCtx final
{
public:
ObTenantFreezeCtx();
~ObTenantFreezeCtx() { reset(); }
void reset();
public:
// snapshot of tenant_info
int64_t mem_lower_limit_;
int64_t mem_upper_limit_;
int64_t mem_memstore_limit_;
// running data
int64_t memstore_freeze_trigger_;
int64_t max_mem_memstore_can_get_now_;
int64_t kvcache_mem_;
int64_t active_memstore_used_;
int64_t total_memstore_used_;
int64_t total_memstore_hold_;
private:
DISABLE_COPY_ASSIGN(ObTenantFreezeCtx);
};
// store the tenant info, such as memory limit, memstore limit,
// slow freeze flag, freezing flag and so on.
class ObTenantInfo : public ObDLinkBase<ObTenantInfo>
@ -59,13 +85,14 @@ public:
void reset();
int update_frozen_scn(int64_t frozen_scn);
int64_t mem_memstore_left() const;
void update_mem_limit(const int64_t lower_limit, const int64_t upper_limit);
void get_mem_limit(int64_t &lower_limit, int64_t &upper_limit) const;
void update_memstore_limit(const int64_t memstore_limit_percentage);
int64_t get_memstore_limit() const;
void get_freeze_ctx(ObTenantFreezeCtx &ctx) const;
public:
uint64_t tenant_id_;
int64_t mem_lower_limit_; // the min memory limit
int64_t mem_upper_limit_; // the max memory limit
// mem_memstore_limit will be checked when **leader** partitions
// perform writing operation (select for update is included)
int64_t mem_memstore_limit_; // the max memstore limit
bool is_loaded_; // whether the memory limit set or not.
bool is_freezing_; // is the tenant freezing now.
int64_t last_freeze_clock_;
@ -76,6 +103,15 @@ public:
int64_t slow_freeze_timestamp_; // the last slow freeze time timestamp
int64_t slow_freeze_min_protect_clock_;
common::ObTabletID slow_tablet_;
private:
// protect mem_lower_limit_/mem_upper_limit_/mem_memstore_limit_
// to make sure it is consistency
SpinRWLock lock_;
int64_t mem_lower_limit_; // the min memory limit
int64_t mem_upper_limit_; // the max memory limit
// mem_memstore_limit will be checked when **leader** partitions
// perform writing operation (select for update is included)
int64_t mem_memstore_limit_; // the max memstore limit
private:
DISALLOW_COPY_AND_ASSIGN(ObTenantInfo);
};