diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index d3e6250c6..18cd0f2d5 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -204,7 +204,7 @@ int ObTenantFreezer::ls_freeze_(ObLS *ls) return ret; } -int ObTenantFreezer::tenant_freeze() +int ObTenantFreezer::tenant_freeze_() { int ret = OB_SUCCESS; int first_fail_ret = OB_SUCCESS; @@ -212,10 +212,7 @@ int ObTenantFreezer::tenant_freeze() ObLSService *ls_srv = MTL(ObLSService *); FLOG_INFO("[TenantFreezer] tenant_freeze start", KR(ret)); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("[TenantFreezer] tenant freezer not inited", KR(ret)); - } else if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { + if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret)); } else { ObLS *ls = nullptr; @@ -349,48 +346,34 @@ int ObTenantFreezer::get_tenant_tx_data_mem_used_(int64_t &tenant_tx_data_mem_us return ret; } -int ObTenantFreezer::check_and_freeze_normal_data_() +int ObTenantFreezer::check_and_freeze_normal_data_(ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; bool upgrade_mode = GCONF.in_major_version_upgrade_mode(); int tmp_ret = OB_SUCCESS; bool need_freeze = false; - int64_t active_memstore_used = 0; - int64_t total_memstore_used = 0; - int64_t total_memstore_hold = 0; - int64_t memstore_freeze_trigger = 0; if (OB_UNLIKELY(upgrade_mode)) { // skip trigger freeze while upgrading } else { - { - SpinRLockGuard guard(lock_); - if (!tenant_info_.is_loaded_) { - // do nothing - } else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) { - LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret)); - } else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used, - total_memstore_used, - total_memstore_hold))) { - LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret)); - } else { - need_freeze = need_freeze_(active_memstore_used, - memstore_freeze_trigger); - if (need_freeze && !is_minor_need_slow_(total_memstore_hold, memstore_freeze_trigger)) { - unset_tenant_slow_freeze_(); - } - log_frozen_memstore_info_if_need_(active_memstore_used, total_memstore_used, - total_memstore_hold, memstore_freeze_trigger); - halt_prewarm_if_need_(memstore_freeze_trigger, total_memstore_hold); + if (OB_FAIL(get_freeze_trigger_(ctx))) { + LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret)); + } else if (OB_FAIL(get_tenant_mem_usage_(ctx))) { + LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret)); + } else { + need_freeze = need_freeze_(ctx); + if (need_freeze && !is_minor_need_slow_(ctx)) { + unset_tenant_slow_freeze_(); } + log_frozen_memstore_info_if_need_(ctx); + halt_prewarm_if_need_(ctx); } // must out of the lock, to make sure there is no deadlock, just because of tenant freeze hung. if (OB_TMP_FAIL(do_major_if_need_(need_freeze))) { LOG_WARN("[TenantFreezer] fail to do major freeze", K(tmp_ret)); } if (need_freeze) { - if (OB_TMP_FAIL(do_minor_freeze_(active_memstore_used, - memstore_freeze_trigger))) { + if (OB_TMP_FAIL(do_minor_freeze_(ctx))) { LOG_WARN("[TenantFreezer] fail to do minor freeze", K(tmp_ret)); } } @@ -444,11 +427,15 @@ int ObTenantFreezer::check_and_do_freeze() int ret = OB_SUCCESS; int64_t check_and_freeze_start_ts = ObTimeUtil::current_time(); + ObTenantFreezeCtx ctx; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); - } else if (OB_FAIL(check_and_freeze_normal_data_())) { + } else if (!tenant_info_.is_loaded_) { + // do nothing + } else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) { + } else if (OB_FAIL(check_and_freeze_normal_data_(ctx))) { LOG_WARN("[TenantFreezer] check and freeze normal data failed.", KR(ret)); } else if (OB_FAIL(check_and_freeze_tx_data_())) { LOG_WARN("[TenantFreezer] check and freeze tx data failed.", KR(ret)); @@ -477,27 +464,25 @@ int ObTenantFreezer::retry_failed_major_freeze_(bool &triggered) return ret; } -int ObTenantFreezer::set_tenant_freezing() +int ObTenantFreezer::set_tenant_freezing_() { int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { - SpinRLockGuard guard(lock_); ATOMIC_AAF(&tenant_info_.freeze_cnt_, 1); } return ret; } -int ObTenantFreezer::unset_tenant_freezing(const bool rollback_freeze_cnt) +int ObTenantFreezer::unset_tenant_freezing_(const bool rollback_freeze_cnt) { int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { - SpinRLockGuard guard(lock_); if (rollback_freeze_cnt) { if (ATOMIC_AAF(&tenant_info_.freeze_cnt_, -1) < 0) { tenant_info_.freeze_cnt_ = 0; @@ -517,7 +502,6 @@ int ObTenantFreezer::set_tenant_slow_freeze( LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { const uint64_t tenant_id = tenant_info_.tenant_id_; - SpinRLockGuard guard(lock_); if (!tenant_info_.slow_freeze_) { bool success = ATOMIC_BCAS(&tenant_info_.slow_freeze_, false, true); if (success) { @@ -561,7 +545,6 @@ int ObTenantFreezer::unset_tenant_slow_freeze() ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { - SpinRLockGuard guard(lock_); ret = unset_tenant_slow_freeze_(); } return ret; @@ -575,7 +558,6 @@ int ObTenantFreezer::unset_tenant_slow_freeze(const common::ObTabletID &tablet_i LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { const uint64_t tenant_id = tenant_info_.tenant_id_; - SpinRLockGuard guard(lock_); if (tenant_info_.slow_freeze_ && tenant_info_.slow_tablet_ == tablet_id) { bool success = ATOMIC_BCAS(&tenant_info_.slow_freeze_, true, false); if (success) { @@ -619,25 +601,25 @@ int ObTenantFreezer::set_tenant_mem_limit( KR(ret)); } else { const uint64_t tenant_id = tenant_info_.tenant_id_; - SpinWLockGuard guard(lock_); // It should be possible to change to a read lock here, this lock is a structural lock, it is not appropriate to borrow - int64_t memstore_freeze_trigger_limit = 0; - tenant_info_.mem_lower_limit_ = lower_limit; - tenant_info_.mem_upper_limit_ = upper_limit; + ObTenantFreezeCtx ctx; + tenant_info_.update_mem_limit(lower_limit, upper_limit); if (NULL != config_) { - int64_t tmp_var = upper_limit / 100; - tenant_info_.mem_memstore_limit_ = tmp_var * config_->memstore_limit_percentage; - if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger_limit))) { + tenant_info_.update_memstore_limit(config_->memstore_limit_percentage); + } + tenant_info_.is_loaded_ = true; + tenant_info_.get_freeze_ctx(ctx); + if (NULL != config_) { + if (OB_FAIL(get_freeze_trigger_(ctx))) { LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id)); } } - tenant_info_.is_loaded_ = true; if (OB_SUCC(ret)) { LOG_INFO("[TenantFreezer] set tenant mem limit", "tenant id", tenant_id, "mem_lower_limit", lower_limit, "mem_upper_limit", upper_limit, - "mem_memstore_limit", tenant_info_.mem_memstore_limit_, - "memstore_freeze_trigger_limit", memstore_freeze_trigger_limit, + "mem_memstore_limit", ctx.mem_memstore_limit_, + "memstore_freeze_trigger_limit", ctx.memstore_freeze_trigger_, "mem_tenant_limit", get_tenant_memory_limit(tenant_info_.tenant_id_), "mem_tenant_hold", get_tenant_memory_hold(tenant_info_.tenant_id_), "mem_memstore_used", get_tenant_memory_hold(tenant_info_.tenant_id_, @@ -660,12 +642,10 @@ int ObTenantFreezer::get_tenant_mem_limit( LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { const uint64_t tenant_id = tenant_info_.tenant_id_; - SpinRLockGuard guard(lock_); if (false == tenant_info_.is_loaded_) { ret = OB_NOT_REGISTERED; } else { - lower_limit = tenant_info_.mem_lower_limit_; - upper_limit = tenant_info_.mem_upper_limit_; + tenant_info_.get_mem_limit(lower_limit, upper_limit); } } return ret; @@ -689,6 +669,7 @@ int ObTenantFreezer::get_tenant_memstore_cond( RLOCAL(int64_t, last_memstore_freeze_trigger); RLOCAL(int64_t, last_memstore_limit); RLOCAL(int64_t, last_freeze_cnt); + ObTenantFreezeCtx ctx; active_memstore_used = 0; total_memstore_used = 0; @@ -706,19 +687,20 @@ int ObTenantFreezer::get_tenant_memstore_cond( memstore_limit = last_memstore_limit; freeze_cnt = last_freeze_cnt; } else { - const uint64_t tenant_id = tenant_info_.tenant_id_; - SpinRLockGuard guard(lock_); + const uint64_t tenant_id = MTL_ID(); if (false == tenant_info_.is_loaded_) { ret = OB_ENTRY_NOT_EXIST; LOG_INFO("[TenantFreezer] This tenant not exist", K(tenant_id), KR(ret)); - } else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used, - total_memstore_used, - unused))) { + } else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) { + } else if (OB_FAIL(get_tenant_mem_usage_(ctx))) { LOG_WARN("[TenantFreezer] failed to get tenant mem usage", KR(ret), K(tenant_id)); - } else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) { - LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id)); + } else if (OB_FAIL(get_freeze_trigger_(ctx))) { + LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id)); } else { - memstore_limit = tenant_info_.mem_memstore_limit_; + memstore_limit = ctx.mem_memstore_limit_; + active_memstore_used = ctx.active_memstore_used_; + total_memstore_used = ctx.total_memstore_used_; + memstore_freeze_trigger = ctx.memstore_freeze_trigger_; freeze_cnt = tenant_info_.freeze_cnt_; // cache the result @@ -742,56 +724,44 @@ int ObTenantFreezer::get_tenant_memstore_limit(int64_t &mem_limit) LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); } else { const uint64_t tenant_id = tenant_info_.tenant_id_; - SpinRLockGuard guard(lock_); if (false == tenant_info_.is_loaded_) { mem_limit = INT64_MAX; LOG_INFO("[TenantFreezer] This tenant not exist", K(tenant_id), KR(ret)); } else { - mem_limit = tenant_info_.mem_memstore_limit_; + mem_limit = tenant_info_.get_memstore_limit(); } } return ret; } -int ObTenantFreezer::get_tenant_mem_usage_( - int64_t &active_memstore_used, - int64_t &total_memstore_used, - int64_t &total_memstore_hold) +int ObTenantFreezer::get_tenant_mem_usage_(ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; ObTenantMemstoreAllocator *tenant_allocator = NULL; - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); + int64_t active_memstore_used = 0; + int64_t total_memstore_used = 0; + int64_t total_memstore_hold = 0; + + const uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id, + tenant_allocator))) { + LOG_WARN("[TenantFreezer] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id)); + } else if (NULL == tenant_allocator) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("[TenantFreezer] tenant memstore allocator is NULL", KR(ret), K(tenant_id)); } else { - const uint64_t tenant_id = tenant_info_.tenant_id_; - if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id, - tenant_allocator))) { - LOG_WARN("[TenantFreezer] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id)); - } else if (NULL == tenant_allocator) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("[TenantFreezer] tenant memstore allocator is NULL", KR(ret), K(tenant_id)); - } else { - active_memstore_used = tenant_allocator->get_mem_active_memstore_used(); - total_memstore_used = tenant_allocator->get_mem_total_memstore_used(); - total_memstore_hold = get_tenant_memory_hold(tenant_id, - ObCtxIds::MEMSTORE_CTX_ID); - } + active_memstore_used = tenant_allocator->get_mem_active_memstore_used(); + total_memstore_used = tenant_allocator->get_mem_total_memstore_used(); + total_memstore_hold = get_tenant_memory_hold(tenant_id, + ObCtxIds::MEMSTORE_CTX_ID); } + ctx.active_memstore_used_ = active_memstore_used; + ctx.total_memstore_used_ = total_memstore_used; + ctx.total_memstore_hold_ = total_memstore_hold; return ret; } -int ObTenantFreezer::get_freeze_trigger_(int64_t &memstore_freeze_trigger) -{ - int64_t not_used = 0; - int64_t not_used2 = 0; - - return get_freeze_trigger_(not_used, - not_used2, - memstore_freeze_trigger); -} - static inline bool is_add_overflow(int64_t first, int64_t second, int64_t &res) { if (first + second < 0) { @@ -802,41 +772,18 @@ static inline bool is_add_overflow(int64_t first, int64_t second, int64_t &res) } } -int ObTenantFreezer::get_mem_remain_trigger_( - int64_t &mem_remain_trigger) -{ - int ret = OB_SUCCESS; - const int64_t tenant_id = tenant_info_.tenant_id_; - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); - double memstore_limit = tenant_info_.mem_memstore_limit_; - - // 1. trigger by write throttling - if (!tenant_config.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("[TenantFreezer] failed to get tenant config", KR(ret)); - } else { - int64_t trigger_percentage = tenant_config->writing_throttling_trigger_percentage; - mem_remain_trigger = memstore_limit * (100 - trigger_percentage) / 100 / 0.95; - } - return ret; -} - -int ObTenantFreezer::get_freeze_trigger_( - /* Now the maximum memory size that the memstore module can preempt and obtain */ - int64_t &max_mem_memstore_can_get_now, - int64_t &kv_cache_mem, - int64_t &memstore_freeze_trigger) +int ObTenantFreezer::get_freeze_trigger_(ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; ObTenantResourceMgrHandle resource_handle; - const uint64_t tenant_id = tenant_info_.tenant_id_; - const int64_t mem_memstore_limit = tenant_info_.mem_memstore_limit_; - if (OB_UNLIKELY(NULL == config_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("[TenantFreezer] config_ is nullptr", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ObResourceMgr::get_instance(). - get_tenant_resource_mgr(tenant_id, - resource_handle))) { + const uint64_t tenant_id = MTL_ID(); + const int64_t mem_memstore_limit = ctx.mem_memstore_limit_; + int64_t kv_cache_mem = 0; + int64_t memstore_freeze_trigger = 0; + int64_t max_mem_memstore_can_get_now = 0; + if (OB_FAIL(ObResourceMgr::get_instance(). + get_tenant_resource_mgr(tenant_id, + resource_handle))) { LOG_WARN("[TenantFreezer] fail to get resource mgr", KR(ret), K(tenant_id)); ret = OB_SUCCESS; memstore_freeze_trigger = @@ -881,6 +828,10 @@ int ObTenantFreezer::get_freeze_trigger_( memstore_freeze_trigger = min / 100 * get_freeze_trigger_percentage_(); } } + // result + ctx.max_mem_memstore_can_get_now_ = max_mem_memstore_can_get_now; + ctx.memstore_freeze_trigger_ = memstore_freeze_trigger; + ctx.kvcache_mem_ = kv_cache_mem; return ret; } @@ -892,6 +843,7 @@ int ObTenantFreezer::check_tenant_out_of_memstore_limit(bool &is_out_of_mem) RLOCAL(int64_t, last_check_timestamp); RLOCAL(bool, last_result); int64_t current_time = OB_TSC_TIMESTAMP.current_time(); + ObTenantFreezeCtx ctx; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); @@ -902,19 +854,14 @@ int ObTenantFreezer::check_tenant_out_of_memstore_limit(bool &is_out_of_mem) // Check once when the last memory burst or tenant_id does not match or the interval reaches the threshold is_out_of_mem = false; } else { - int64_t active_memstore_used = 0; - int64_t total_memstore_used = 0; - int64_t total_memstore_hold = 0; - SpinRLockGuard guard(lock_); if (false == tenant_info_.is_loaded_) { is_out_of_mem = false; LOG_INFO("[TenantFreezer] This tenant not exist", K(tenant_id), KR(ret)); - } else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used, - total_memstore_used, - total_memstore_hold))) { + } else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) { + } else if (OB_FAIL(get_tenant_mem_usage_(ctx))) { LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret), K(tenant_info_.tenant_id_)); } else { - is_out_of_mem = (total_memstore_hold > tenant_info_.mem_memstore_limit_); + is_out_of_mem = (ctx.total_memstore_hold_ > ctx.mem_memstore_limit_); } last_check_timestamp = current_time; } @@ -930,32 +877,26 @@ bool ObTenantFreezer::tenant_need_major_freeze() { int ret = OB_SUCCESS; bool bool_ret = false; - int64_t active_memstore_used = 0; - int64_t total_memstore_used = 0; - int64_t total_memstore_hold = 0; - int64_t memstore_freeze_trigger = 0; + ObTenantFreezeCtx ctx; if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("tenant manager not init", K(ret)); } else { - SpinRLockGuard guard(lock_); if (!tenant_info_.is_loaded_) { // do nothing - } else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) { + } else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) { + } else if (OB_FAIL(get_freeze_trigger_(ctx))) { LOG_WARN("fail to get minor freeze trigger", K(ret), K(tenant_info_.tenant_id_)); - } else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used, - total_memstore_used, - total_memstore_hold))) { + } else if (OB_FAIL(get_tenant_mem_usage_(ctx))) { LOG_WARN("fail to get mem usage", K(ret), K(tenant_info_.tenant_id_)); } else { - bool_ret = need_freeze_(active_memstore_used, - memstore_freeze_trigger); + bool_ret = need_freeze_(ctx); if (bool_ret) { LOG_INFO("A major freeze is needed", "active_memstore_used_", - active_memstore_used, + ctx.active_memstore_used_, "memstore_freeze_trigger_limit_", - memstore_freeze_trigger, + ctx.memstore_freeze_trigger_, "tenant_id", tenant_info_.tenant_id_); } @@ -964,7 +905,7 @@ bool ObTenantFreezer::tenant_need_major_freeze() return bool_ret; } -int64_t ObTenantFreezer::get_freeze_trigger_percentage_() const +int64_t ObTenantFreezer::get_freeze_trigger_percentage_() { static const int64_t DEFAULT_FREEZE_TRIGGER_PERCENTAGE = 20; int64_t percent = DEFAULT_FREEZE_TRIGGER_PERCENTAGE; @@ -1041,11 +982,8 @@ void ObTenantFreezer::reload_config() freeze_trigger_percentage, KR(ret)); } else { - SpinWLockGuard guard(lock_); // It should be possible to change to a read lock here, this lock is a structural lock, it is not appropriate to borrow if (true == tenant_info_.is_loaded_) { - int64_t tmp_var = tenant_info_.mem_upper_limit_ / 100; - tenant_info_.mem_memstore_limit_ = - tmp_var * config_->memstore_limit_percentage; + tenant_info_.update_memstore_limit(config_->memstore_limit_percentage); } } if (OB_SUCCESS == ret) { @@ -1063,25 +1001,16 @@ int ObTenantFreezer::print_tenant_usage( int64_t &pos) { int ret = OB_SUCCESS; + ObTenantFreezeCtx ctx; lib::ObMallocAllocator *mallocator = lib::ObMallocAllocator::get_instance(); - int64_t active_memstore_used = 0; - int64_t total_memstore_used = 0; - int64_t total_memstore_hold = 0; - int64_t memstore_freeze_trigger_limit = 0; - int64_t max_mem_memstore_can_get_now = 0; - int64_t kv_cache_mem = 0; - SpinWLockGuard guard(lock_); if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); - } else if (OB_FAIL(get_tenant_mem_usage_(active_memstore_used, - total_memstore_used, - total_memstore_hold))) { + } else if (FALSE_IT(tenant_info_.get_freeze_ctx(ctx))) { + } else if (OB_FAIL(get_tenant_mem_usage_(ctx))) { LOG_WARN("[TenantFreezer] fail to get mem usage", KR(ret), K(tenant_info_.tenant_id_)); - } else if (OB_FAIL(get_freeze_trigger_(max_mem_memstore_can_get_now, - kv_cache_mem, - memstore_freeze_trigger_limit))) { + } else if (OB_FAIL(get_freeze_trigger_(ctx))) { LOG_WARN("[TenantFreezer] get tenant minor freeze trigger error", KR(ret), K(tenant_info_.tenant_id_)); } else { ret = databuff_printf(print_buf, buf_len, pos, @@ -1094,21 +1023,18 @@ int ObTenantFreezer::print_tenant_usage( "memstore_limit=% '15ld " "mem_tenant_limit=% '15ld " "mem_tenant_hold=% '15ld " - "mem_memstore_used=% '15ld " "kv_cache_mem=% '15ld " "max_mem_memstore_can_get_now=% '15ld\n", tenant_info_.tenant_id_, - active_memstore_used, - total_memstore_used, - total_memstore_hold, - memstore_freeze_trigger_limit, - tenant_info_.mem_memstore_limit_, + ctx.active_memstore_used_, + ctx.total_memstore_used_, + ctx.total_memstore_hold_, + ctx.memstore_freeze_trigger_, + ctx.mem_memstore_limit_, get_tenant_memory_limit(tenant_info_.tenant_id_), get_tenant_memory_hold(tenant_info_.tenant_id_), - get_tenant_memory_hold(tenant_info_.tenant_id_, - ObCtxIds::MEMSTORE_CTX_ID), - kv_cache_mem, - max_mem_memstore_can_get_now); + ctx.kvcache_mem_, + ctx.max_mem_memstore_can_get_now_); } if (!OB_ISNULL(mallocator)) { @@ -1134,17 +1060,14 @@ int ObTenantFreezer::get_global_frozen_scn_(int64_t &frozen_scn) return ret; } -bool ObTenantFreezer::need_freeze_( - const int64_t active_memstore_used, - const int64_t memstore_freeze_trigger) +bool ObTenantFreezer::need_freeze_(const ObTenantFreezeCtx &ctx) { bool need_freeze = false; - const int64_t tenant_id = tenant_info_.tenant_id_; // 1. trigger by active memstore used. - if (active_memstore_used > memstore_freeze_trigger) { + if (ctx.active_memstore_used_ > ctx.memstore_freeze_trigger_) { need_freeze = true; LOG_INFO("[TenantFreezer] A minor freeze is needed by active memstore used.", - K(active_memstore_used), K(memstore_freeze_trigger), K(tenant_id)); + K(ctx.active_memstore_used_), K(ctx.memstore_freeze_trigger_)); } return need_freeze; } @@ -1160,16 +1083,14 @@ bool ObTenantFreezer::is_major_freeze_turn_() return (major_compact_trigger != 0 && freeze_cnt >= major_compact_trigger); } -bool ObTenantFreezer::is_minor_need_slow_( - const int64_t total_memstore_hold, - const int64_t memstore_freeze_trigger) +bool ObTenantFreezer::is_minor_need_slow_(const ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; bool need_slow = false; if (tenant_info_.slow_freeze_) { need_slow = true; int64_t now = ObTimeUtility::fast_current_time(); - if (total_memstore_hold <= memstore_freeze_trigger) { + if (ctx.total_memstore_hold_ <= ctx.memstore_freeze_trigger_) { // no need minor freeze } else if (now - tenant_info_.slow_freeze_timestamp_ >= SLOW_FREEZE_INTERVAL) { need_slow = false; @@ -1180,15 +1101,14 @@ bool ObTenantFreezer::is_minor_need_slow_( return need_slow; } -int ObTenantFreezer::do_minor_freeze_(const int64_t active_memstore_used, - const int64_t memstore_freeze_trigger) +int ObTenantFreezer::do_minor_freeze_(const ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; bool rollback_freeze_cnt = false; LOG_INFO("[TenantFreezer] A minor freeze is needed", - "active_memstore_used_", active_memstore_used, - "memstore_freeze_trigger", memstore_freeze_trigger, + "active_memstore_used_", ctx.active_memstore_used_, + "memstore_freeze_trigger", ctx.memstore_freeze_trigger_, "mem_tenant_remain", get_tenant_memory_remain(MTL_ID()), "mem_tenant_limit", get_tenant_memory_limit(MTL_ID()), "mem_tenant_hold", get_tenant_memory_hold(MTL_ID()), @@ -1196,10 +1116,10 @@ int ObTenantFreezer::do_minor_freeze_(const int64_t active_memstore_used, ObCtxIds::MEMSTORE_CTX_ID), "tenant_id", MTL_ID()); - if (OB_FAIL(set_tenant_freezing())) { + if (OB_FAIL(set_tenant_freezing_())) { } else { bool rollback_freeze_cnt = false; - if (OB_FAIL(tenant_freeze())) { + if (OB_FAIL(tenant_freeze_())) { rollback_freeze_cnt = true; LOG_WARN("fail to minor freeze", K(ret)); } else { @@ -1208,7 +1128,7 @@ int ObTenantFreezer::do_minor_freeze_(const int64_t active_memstore_used, // clear freezing mark for tenant int tmp_ret = OB_SUCCESS; if (OB_UNLIKELY(OB_SUCCESS != - (tmp_ret = unset_tenant_freezing(rollback_freeze_cnt)))) { + (tmp_ret = unset_tenant_freezing_(rollback_freeze_cnt)))) { LOG_WARN("unset tenant freezing mark failed", K(tmp_ret)); if (OB_SUCC(ret)) { ret = tmp_ret; @@ -1230,21 +1150,18 @@ int ObTenantFreezer::do_major_if_need_(const bool need_freeze) if (OB_TMP_FAIL(retry_failed_major_freeze_(major_triggered))) { LOG_WARN("fail to do major freeze due to previous failure", K(tmp_ret)); } - // update frozen scn - if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) { + if (!tenant_info_.is_loaded_) { + // do nothing + // update frozen scn + } else if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) { LOG_WARN("fail to get global frozen version", K(ret)); + } else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) { + LOG_WARN("fail to update frozen version", K(ret), K(frozen_scn), K_(tenant_info)); } else { - SpinRLockGuard guard(lock_); - if (!tenant_info_.is_loaded_) { - // do nothing - } else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) { - LOG_WARN("fail to update frozen version", K(ret), K(frozen_scn), K_(tenant_info)); - } else { - need_major = (need_freeze && - !major_triggered && - is_major_freeze_turn_()); - curr_frozen_scn = tenant_info_.frozen_scn_; - } + need_major = (need_freeze && + !major_triggered && + is_major_freeze_turn_()); + curr_frozen_scn = tenant_info_.frozen_scn_; } if (need_major) { if (OB_FAIL(do_major_freeze_(curr_frozen_scn))) { @@ -1268,22 +1185,18 @@ int ObTenantFreezer::do_major_freeze_(const int64_t try_frozen_scn) return ret; } -void ObTenantFreezer::log_frozen_memstore_info_if_need_( - const int64_t active_memstore_used, - const int64_t total_memstore_used, - const int64_t total_memstore_hold, - const int64_t memstore_freeze_trigger) +void ObTenantFreezer::log_frozen_memstore_info_if_need_(const ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; ObTenantMemstoreAllocator *tenant_allocator = NULL; - if (total_memstore_hold > memstore_freeze_trigger) { + if (ctx.total_memstore_hold_ > ctx.memstore_freeze_trigger_) { // There is an unreleased memstable LOG_INFO("[TenantFreezer] tenant have inactive memstores", - K(active_memstore_used), - K(total_memstore_used), - K(total_memstore_hold), + K(ctx.active_memstore_used_), + K(ctx.total_memstore_used_), + K(ctx.total_memstore_hold_), "memstore_freeze_trigger_limit_", - memstore_freeze_trigger, + ctx.memstore_freeze_trigger_, "tenant_id", MTL_ID()); @@ -1299,15 +1212,13 @@ void ObTenantFreezer::log_frozen_memstore_info_if_need_( } } -void ObTenantFreezer::halt_prewarm_if_need_( - const int64_t memstore_freeze_trigger, - const int64_t total_memstore_hold) +void ObTenantFreezer::halt_prewarm_if_need_(const ObTenantFreezeCtx &ctx) { int ret = OB_SUCCESS; // When the memory is tight, try to abort the warm-up to release memstore - int64_t mem_danger_limit = tenant_info_.mem_memstore_limit_ - - ((tenant_info_.mem_memstore_limit_ - memstore_freeze_trigger) >> 2); - if (total_memstore_hold > mem_danger_limit) { + int64_t mem_danger_limit = ctx.mem_memstore_limit_ + - ((ctx.mem_memstore_limit_ - ctx.memstore_freeze_trigger_) >> 2); + if (ctx.total_memstore_hold_ > mem_danger_limit) { int64_t curr_ts = ObTimeUtility::current_time(); if (curr_ts - tenant_info_.last_halt_ts_ > 10L * 1000L * 1000L) { if (OB_FAIL(svr_rpc_proxy_->to(self_). diff --git a/src/storage/tx_storage/ob_tenant_freezer.h b/src/storage/tx_storage/ob_tenant_freezer.h index df0067b52..b6f1f7778 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.h +++ b/src/storage/tx_storage/ob_tenant_freezer.h @@ -52,20 +52,11 @@ public: int stop(); void wait(); - // freeze all the ls of this tenant. - // return the first failed code. - int tenant_freeze(); // freeze a tablet int tablet_freeze(const common::ObTabletID &tablet_id, const bool is_force_freeze=false); // check if this tenant's memstore is out of range, and trigger minor/major freeze. int check_and_do_freeze(); - // we can only deal with freeze one by one. - // set tenant freezing will prevent a new freeze. - int set_tenant_freezing(); - // unset tenant freezing flag. - // @param[in] rollback_freeze_cnt, reduce the tenant's freeze count by 1, if true. - int unset_tenant_freezing(const bool rollback_freeze_cnt); // If the tenant's freeze process is slowed, we will only freeze one time every // SLOW_FREEZE_INTERVAL. @@ -128,45 +119,40 @@ public: ObServerConfig *get_config() { return config_; } bool exist_ls_freezing(); private: - int ls_freeze_(ObLS *ls); - int64_t get_freeze_trigger_percentage_() const; + static int ls_freeze_(ObLS *ls); + // freeze all the ls of this tenant. + // return the first failed code. + static int tenant_freeze_(); + // we can only deal with freeze one by one. + // set tenant freezing will prevent a new freeze. + int set_tenant_freezing_(); + // unset tenant freezing flag. + // @param[in] rollback_freeze_cnt, reduce the tenant's freeze count by 1, if true. + int unset_tenant_freezing_(const bool rollback_freeze_cnt); + static int64_t get_freeze_trigger_percentage_(); int post_freeze_request_(const storage::ObFreezeType freeze_type, const int64_t try_frozen_version); int retry_failed_major_freeze_(bool &triggered); int get_global_frozen_scn_(int64_t &frozen_version); int post_tx_data_freeze_request_(); - int get_tenant_mem_usage_(int64_t &active_memstore_used, - int64_t &total_memstore_used, - int64_t &total_memstore_hold); - int get_freeze_trigger_(int64_t &memstore_freeze_trigger); - int get_freeze_trigger_(int64_t &max_mem_memstore_can_get_now, - int64_t &kvcache_mem, - int64_t &memstore_freeze_trigger); - int get_mem_remain_trigger_(int64_t &mem_remain_trigger); - bool need_freeze_(const int64_t active_memstore_used, - const int64_t memstore_freeze_trigger); - bool is_minor_need_slow_(const int64_t mem_total_memstore_hold, - const int64_t memstore_freeze_trigger); + int get_tenant_mem_usage_(ObTenantFreezeCtx &ctx); + static int get_freeze_trigger_(ObTenantFreezeCtx &ctx); + static bool need_freeze_(const ObTenantFreezeCtx &ctx); + bool is_minor_need_slow_(const ObTenantFreezeCtx &ctx); bool is_major_freeze_turn_(); int do_major_if_need_(const bool need_freeze); - int do_minor_freeze_(const int64_t active_memstore_used, - const int64_t memstore_freeze_trigger); + int do_minor_freeze_(const ObTenantFreezeCtx &ctx); int do_major_freeze_(const int64_t try_frozen_scn); - void log_frozen_memstore_info_if_need_(const int64_t active_memstore_used, - const int64_t mem_total_memstore_used, - const int64_t mem_total_memstore_hold, - const int64_t memstore_freeze_trigger); - void halt_prewarm_if_need_(const int64_t memstore_freeze_trigger, - const int64_t mem_total_memstore_hold); + void log_frozen_memstore_info_if_need_(const ObTenantFreezeCtx &ctx); + void halt_prewarm_if_need_(const ObTenantFreezeCtx &ctx); int unset_tenant_slow_freeze_(); - int check_and_freeze_normal_data_(); + int check_and_freeze_normal_data_(ObTenantFreezeCtx &ctx); int check_and_freeze_tx_data_(); int get_tenant_tx_data_mem_used_(int64_t &tenant_tx_data_mem_used); int get_ls_tx_data_mem_used_(ObLS *ls, int64_t &ls_tx_data_mem_used); private: bool is_inited_; bool is_freezing_tx_data_; - SpinRWLock lock_; ObTenantInfo tenant_info_; // store the mem limit, memstore limit and etc. obrpc::ObTenantFreezerRpcProxy rpc_proxy_; // used to trigger minor/major freeze obrpc::ObTenantFreezerRpcCb tenant_mgr_cb_; // callback after the trigger rpc finish. diff --git a/src/storage/tx_storage/ob_tenant_freezer_common.cpp b/src/storage/tx_storage/ob_tenant_freezer_common.cpp index 5ceffc7f2..132284e3f 100644 --- a/src/storage/tx_storage/ob_tenant_freezer_common.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer_common.cpp @@ -30,11 +30,34 @@ OB_SERIALIZE_MEMBER(ObTenantFreezeArg, freeze_type_, try_frozen_scn_); -ObTenantInfo::ObTenantInfo() - : tenant_id_(INT64_MAX), - mem_lower_limit_(0), +ObTenantFreezeCtx::ObTenantFreezeCtx() + : mem_lower_limit_(0), mem_upper_limit_(0), mem_memstore_limit_(0), + memstore_freeze_trigger_(0), + max_mem_memstore_can_get_now_(0), + kvcache_mem_(0), + active_memstore_used_(0), + total_memstore_used_(0), + total_memstore_hold_(0) +{ +} + +void ObTenantFreezeCtx::reset() +{ + mem_lower_limit_ = 0; + mem_upper_limit_ = 0; + mem_memstore_limit_ = 0; + memstore_freeze_trigger_ = 0; + max_mem_memstore_can_get_now_ = 0; + kvcache_mem_ = 0; + active_memstore_used_ = 0; + total_memstore_used_ = 0; + total_memstore_hold_ = 0; +} + +ObTenantInfo::ObTenantInfo() + : tenant_id_(INT64_MAX), is_loaded_(false), is_freezing_(false), last_freeze_clock_(0), @@ -43,16 +66,16 @@ ObTenantInfo::ObTenantInfo() last_halt_ts_(0), slow_freeze_(false), slow_freeze_timestamp_(0), - slow_freeze_min_protect_clock_(INT64_MAX) + slow_freeze_min_protect_clock_(INT64_MAX), + mem_lower_limit_(0), + mem_upper_limit_(0), + mem_memstore_limit_(0) { } void ObTenantInfo::reset() { tenant_id_ = OB_INVALID_TENANT_ID; // i64 max as invalid. - mem_memstore_limit_ = 0; - mem_lower_limit_ = 0; - mem_upper_limit_ = 0; is_loaded_ = false; is_freezing_ = false; frozen_scn_ = 0; @@ -62,6 +85,9 @@ void ObTenantInfo::reset() slow_freeze_timestamp_ = 0; slow_freeze_min_protect_clock_ = INT64_MAX; slow_tablet_.reset(); + mem_memstore_limit_ = 0; + mem_lower_limit_ = 0; + mem_upper_limit_ = 0; } int ObTenantInfo::update_frozen_scn(int64_t frozen_scn) @@ -82,5 +108,41 @@ int64_t ObTenantInfo::mem_memstore_left() const return max(0, mem_memstore_limit_ - (int64_t)memstore_hold); } +void ObTenantInfo::get_mem_limit(int64_t &lower_limit, int64_t &upper_limit) const +{ + SpinRLockGuard guard(lock_); + lower_limit = mem_lower_limit_; + upper_limit = mem_upper_limit_; +} + +void ObTenantInfo::update_mem_limit(const int64_t lower_limit, + const int64_t upper_limit) +{ + SpinWLockGuard guard(lock_); + mem_lower_limit_ = lower_limit; + mem_upper_limit_ = upper_limit; +} + +void ObTenantInfo::update_memstore_limit(const int64_t memstore_limit_percentage) +{ + SpinWLockGuard guard(lock_); + int64_t tmp_var = mem_upper_limit_ / 100; + mem_memstore_limit_ = tmp_var * memstore_limit_percentage; +} + +int64_t ObTenantInfo::get_memstore_limit() const +{ + SpinRLockGuard guard(lock_); + return mem_memstore_limit_; +} + +void ObTenantInfo::get_freeze_ctx(ObTenantFreezeCtx &ctx) const +{ + SpinRLockGuard guard(lock_); + ctx.mem_lower_limit_ = mem_lower_limit_; + ctx.mem_upper_limit_ = mem_upper_limit_; + ctx.mem_memstore_limit_ = mem_memstore_limit_; +} + } // storage } // oceanbase diff --git a/src/storage/tx_storage/ob_tenant_freezer_common.h b/src/storage/tx_storage/ob_tenant_freezer_common.h index f0069f323..f91925617 100644 --- a/src/storage/tx_storage/ob_tenant_freezer_common.h +++ b/src/storage/tx_storage/ob_tenant_freezer_common.h @@ -49,6 +49,32 @@ struct ObRetryMajorInfo TO_STRING_KV(K_(tenant_id), K_(frozen_scn)); }; +// store a snapshot of the tenant info to make sure +// the tenant_info will be a atomic value +struct ObTenantFreezeCtx final +{ +public: + ObTenantFreezeCtx(); + ~ObTenantFreezeCtx() { reset(); } + void reset(); +public: + // snapshot of tenant_info + int64_t mem_lower_limit_; + int64_t mem_upper_limit_; + int64_t mem_memstore_limit_; + + // running data + int64_t memstore_freeze_trigger_; + int64_t max_mem_memstore_can_get_now_; + int64_t kvcache_mem_; + + int64_t active_memstore_used_; + int64_t total_memstore_used_; + int64_t total_memstore_hold_; +private: + DISABLE_COPY_ASSIGN(ObTenantFreezeCtx); +}; + // store the tenant info, such as memory limit, memstore limit, // slow freeze flag, freezing flag and so on. class ObTenantInfo : public ObDLinkBase @@ -59,13 +85,14 @@ public: void reset(); int update_frozen_scn(int64_t frozen_scn); int64_t mem_memstore_left() const; + + void update_mem_limit(const int64_t lower_limit, const int64_t upper_limit); + void get_mem_limit(int64_t &lower_limit, int64_t &upper_limit) const; + void update_memstore_limit(const int64_t memstore_limit_percentage); + int64_t get_memstore_limit() const; + void get_freeze_ctx(ObTenantFreezeCtx &ctx) const; public: uint64_t tenant_id_; - int64_t mem_lower_limit_; // the min memory limit - int64_t mem_upper_limit_; // the max memory limit - // mem_memstore_limit will be checked when **leader** partitions - // perform writing operation (select for update is included) - int64_t mem_memstore_limit_; // the max memstore limit bool is_loaded_; // whether the memory limit set or not. bool is_freezing_; // is the tenant freezing now. int64_t last_freeze_clock_; @@ -76,6 +103,15 @@ public: int64_t slow_freeze_timestamp_; // the last slow freeze time timestamp int64_t slow_freeze_min_protect_clock_; common::ObTabletID slow_tablet_; +private: + // protect mem_lower_limit_/mem_upper_limit_/mem_memstore_limit_ + // to make sure it is consistency + SpinRWLock lock_; + int64_t mem_lower_limit_; // the min memory limit + int64_t mem_upper_limit_; // the max memory limit + // mem_memstore_limit will be checked when **leader** partitions + // perform writing operation (select for update is included) + int64_t mem_memstore_limit_; // the max memstore limit private: DISALLOW_COPY_AND_ASSIGN(ObTenantInfo); };