From 28a864535f46439cb809f6f692be32477088e2df Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 9 May 2023 18:43:01 +0000 Subject: [PATCH] FREEZE_CHECKER: check whether the tenant memstore is really frozen --- ...virtual_tenant_memstore_allocator_info.cpp | 2 +- .../allocator/ob_gmemstore_allocator.cpp | 16 ++++- src/share/allocator/ob_gmemstore_allocator.h | 28 +++++++- src/share/allocator/ob_handle_list.h | 11 ++++ src/storage/tx_storage/ob_tenant_freezer.cpp | 3 + src/storage/tx_storage/ob_tenant_freezer.h | 2 +- .../tx_storage/ob_tenant_freezer_common.cpp | 66 +++++++++++++++++++ .../tx_storage/ob_tenant_freezer_common.h | 18 +++++ 8 files changed, 140 insertions(+), 6 deletions(-) diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp index 862d04ec9..310f2d4e3 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp @@ -37,7 +37,7 @@ public: memtable::ObMemtable& mt = handle->mt_; ObLSID ls_id; item.protection_clock_ = handle->get_protection_clock(); - item.is_active_ = mt.is_active_memtable(); + item.is_active_ = handle->is_active(); item.ls_id_ = (OB_SUCCESS == mt.get_ls_id(ls_id)) ? ls_id.id() : ObLSID::INVALID_LS_ID; item.tablet_id_ = mt.get_key().tablet_id_.id(); item.scn_range_ = mt.get_scn_range(); diff --git a/src/share/allocator/ob_gmemstore_allocator.cpp b/src/share/allocator/ob_gmemstore_allocator.cpp index 11a3b6c00..5dee3c6b7 100644 --- a/src/share/allocator/ob_gmemstore_allocator.cpp +++ b/src/share/allocator/ob_gmemstore_allocator.cpp @@ -30,7 +30,21 @@ int FrozenMemstoreInfoLogger::operator()(ObDLink* link) int ret = OB_SUCCESS; ObGMemstoreAllocator::AllocHandle* handle = CONTAINER_OF(link, typeof(*handle), total_list_); memtable::ObMemtable& mt = handle->mt_; - if (mt.is_frozen_memtable()) { + if (handle->is_frozen()) { + if (OB_FAIL(databuff_print_obj(buf_, limit_, pos_, mt))) { + } else { + ret = databuff_printf(buf_, limit_, pos_, ","); + } + } + return ret; +} + +int ActiveMemstoreInfoLogger::operator()(ObDLink* link) +{ + int ret = OB_SUCCESS; + ObGMemstoreAllocator::AllocHandle* handle = CONTAINER_OF(link, typeof(*handle), total_list_); + memtable::ObMemtable& mt = handle->mt_; + if (handle->is_active()) { if (OB_FAIL(databuff_print_obj(buf_, limit_, pos_, mt))) { } else { ret = databuff_printf(buf_, limit_, pos_, ","); diff --git a/src/share/allocator/ob_gmemstore_allocator.h b/src/share/allocator/ob_gmemstore_allocator.h index 747c3e60a..3c66b2c91 100644 --- a/src/share/allocator/ob_gmemstore_allocator.h +++ b/src/share/allocator/ob_gmemstore_allocator.h @@ -33,6 +33,17 @@ struct FrozenMemstoreInfoLogger int64_t limit_; int64_t pos_; }; + +struct ActiveMemstoreInfoLogger +{ + ActiveMemstoreInfoLogger(char* buf, int64_t limit): buf_(buf), limit_(limit), pos_(0) {} + ~ActiveMemstoreInfoLogger() {} + int operator()(ObDLink* link); + char* buf_; + int64_t limit_; + int64_t pos_; +}; + class ObGMemstoreAllocator { public: @@ -120,11 +131,11 @@ public: void* alloc(AllocHandle& handle, int64_t size); void set_frozen(AllocHandle& handle); template - int for_each(Func& f) { + int for_each(Func& f, const bool reverse=false) { int ret = common::OB_SUCCESS; ObDLink* iter = NULL; LockGuard guard(lock_); - while(OB_SUCC(ret) && NULL != (iter = hlist_.next(iter))) { + while(OB_SUCC(ret) && NULL != (iter = (reverse ? hlist_.prev(iter) : hlist_.next(iter)))) { ret = f(iter); } return ret; @@ -134,12 +145,23 @@ public: int64_t hazard = hlist_.hazard(); return hazard == INT64_MAX? 0: (arena_.allocated() - hazard); } + int64_t get_frozen_memstore_pos() const { + int64_t hazard = hlist_.hazard(); + return hazard == INT64_MAX? 0: hazard; + } int64_t get_mem_total_memstore_used() const { return arena_.hold(); } void log_frozen_memstore_info(char* buf, int64_t limit) { if (NULL != buf && limit > 0) { FrozenMemstoreInfoLogger logger(buf, limit); buf[0] = 0; - (void)for_each(logger); + (void)for_each(logger, true /* reverse */); + } + } + void log_active_memstore_info(char *buf, int64_t limit) { + if (NULL != buf && limit > 0) { + ActiveMemstoreInfoLogger logger(buf, limit); + buf[0] = 0; + (void)for_each(logger, true /* reverse */); } } public: diff --git a/src/share/allocator/ob_handle_list.h b/src/share/allocator/ob_handle_list.h index f0f9720a9..2b4b6f392 100644 --- a/src/share/allocator/ob_handle_list.h +++ b/src/share/allocator/ob_handle_list.h @@ -45,6 +45,16 @@ public: } return iter; } + DLink* prev(DLink* iter) { + if (NULL == iter) { + iter = &tail_; + } + iter = (DLink*)iter->prev_; + if (&head_ == iter) { + iter = NULL; + } + return iter; + } static void dlink_insert(ObDLink* cur, ObDLink* x) { ObDLink* next = (ObDLink*)cur->next_; @@ -129,6 +139,7 @@ public: bool is_empty() const { return ATOMIC_LOAD(&total_count_) <= 0; } int64_t hazard() const { return ATOMIC_LOAD(&hazard_); } DLink* next(DLink* iter) { return total_list_.next(iter); } + DLink* prev(DLink* iter) { return total_list_.prev(iter); } protected: void set_frozen_(Handle& handle); int64_t alloc_id() { return ATOMIC_AAF(&id_, 1); } diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index ca714c588..4277f4a95 100755 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -316,6 +316,7 @@ int ObTenantFreezer::tenant_freeze_() ObLSService *ls_srv = MTL(ObLSService *); FLOG_INFO("[TenantFreezer] tenant_freeze start", KR(ret)); + ObTenantFreezeGuard freeze_guard(allocator_mgr_, ret); if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) { LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret)); } else { @@ -1362,6 +1363,8 @@ int ObTenantFreezer::do_major_if_need_(const bool need_freeze) // update frozen scn } else if (!need_freeze) { // no need major + } else if (!is_major_freeze_turn_()) { + // do nothing } else if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) { LOG_WARN("fail to get global frozen version", K(ret)); } else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) { diff --git a/src/storage/tx_storage/ob_tenant_freezer.h b/src/storage/tx_storage/ob_tenant_freezer.h index 2694c533d..cd98e3bd2 100755 --- a/src/storage/tx_storage/ob_tenant_freezer.h +++ b/src/storage/tx_storage/ob_tenant_freezer.h @@ -153,7 +153,7 @@ private: const int64_t abs_timeout_ts); // freeze all the ls of this tenant. // return the first failed code. - static int tenant_freeze_(); + int tenant_freeze_(); // we can only deal with freeze one by one. // set tenant freezing will prevent a new freeze. int set_tenant_freezing_(); diff --git a/src/storage/tx_storage/ob_tenant_freezer_common.cpp b/src/storage/tx_storage/ob_tenant_freezer_common.cpp index 132284e3f..b7083cf3e 100644 --- a/src/storage/tx_storage/ob_tenant_freezer_common.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer_common.cpp @@ -10,7 +10,14 @@ * See the Mulan PubL v2 for more details. */ +#define USING_LOG_PREFIX STORAGE + +#include "lib/oblog/ob_log.h" #include "lib/alloc/alloc_func.h" +#include "share/allocator/ob_gmemstore_allocator.h" +#include "share/allocator/ob_memstore_allocator_mgr.h" +#include "share/ob_force_print_log.h" +#include "share/rc/ob_tenant_base.h" #include "storage/tx_storage/ob_tenant_freezer_common.h" namespace oceanbase @@ -18,6 +25,7 @@ namespace oceanbase using namespace lib; namespace storage { +typedef ObMemstoreAllocatorMgr::TAllocator ObTenantMemstoreAllocator; DEF_TO_STRING(ObTenantFreezeArg) { @@ -144,5 +152,63 @@ void ObTenantInfo::get_freeze_ctx(ObTenantFreezeCtx &ctx) const ctx.mem_memstore_limit_ = mem_memstore_limit_; } +ObTenantFreezeGuard::ObTenantFreezeGuard(common::ObMemstoreAllocatorMgr *allocator_mgr, + int &err_code, + const int64_t warn_threshold) + : allocator_mgr_(nullptr), + pre_retire_pos_(0), + error_code_(err_code), + time_guard_("FREEZE_CHECKER", warn_threshold) +{ + int ret = OB_SUCCESS; + ObTenantMemstoreAllocator *tenant_allocator = NULL; + const uint64_t tenant_id = MTL_ID(); + if (OB_NOT_NULL(allocator_mgr)) { + allocator_mgr_ = allocator_mgr; + if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id, + tenant_allocator))) { + LOG_WARN("[FREEZE_CHECKER] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id)); + } else if (NULL == tenant_allocator) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("[FREEZE_CHECKER] tenant memstore allocator is NULL", KR(ret), K(tenant_id)); + } else { + pre_retire_pos_ = tenant_allocator->get_retire_clock(); + } + } +} + +ObTenantFreezeGuard::~ObTenantFreezeGuard() +{ + int ret = OB_SUCCESS; + ObTenantMemstoreAllocator *tenant_allocator = NULL; + const uint64_t tenant_id = MTL_ID(); + int64_t curr_frozen_pos = 0; + if (OB_ISNULL(allocator_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("[FREEZE_CHECKER]freeze guard invalid", KR(ret), K_(allocator_mgr), K(lbt())); + } else if (OB_FAIL(error_code_)) { + LOG_WARN("[FREEZE_CHECKER]tenant freeze failed, skip check frozen memstore", KR(ret)); + } else if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id, + tenant_allocator))) { + LOG_WARN("[FREEZE_CHECKER] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id)); + } else if (NULL == tenant_allocator) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("[FREEZE_CHECKER] tenant memstore allocator is NULL", KR(ret), K(tenant_id)); + } else { + curr_frozen_pos = tenant_allocator->get_frozen_memstore_pos(); + const bool retired_mem_frozen = (curr_frozen_pos >= pre_retire_pos_); + const bool has_no_active_memtable = (curr_frozen_pos == 0); + if (!(retired_mem_frozen || has_no_active_memtable)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("[FREEZE_CHECKER]there may be frequent tenant freeze", KR(ret), K(curr_frozen_pos), + K_(pre_retire_pos), K(retired_mem_frozen), K(has_no_active_memtable)); + char active_mt_info[DEFAULT_BUF_LENGTH]; + tenant_allocator->log_active_memstore_info(active_mt_info, + sizeof(active_mt_info)); + FLOG_INFO("[FREEZE_CHECKER] oldest active memtable", "list", active_mt_info); + } + } +} + } // storage } // oceanbase diff --git a/src/storage/tx_storage/ob_tenant_freezer_common.h b/src/storage/tx_storage/ob_tenant_freezer_common.h index f91925617..b941eb4b1 100644 --- a/src/storage/tx_storage/ob_tenant_freezer_common.h +++ b/src/storage/tx_storage/ob_tenant_freezer_common.h @@ -18,6 +18,10 @@ #define OCEABASE_STORAGE_TENANT_FREEZER_COMMON_ namespace oceanbase { +namespace common +{ +class ObMemstoreAllocatorMgr; +} namespace storage { struct ObTenantFreezeArg @@ -116,6 +120,20 @@ private: DISALLOW_COPY_AND_ASSIGN(ObTenantInfo); }; +class ObTenantFreezeGuard +{ +public: + ObTenantFreezeGuard(common::ObMemstoreAllocatorMgr *allocator_mgr, + int &ret, + const int64_t warn_threshold = 60 * 1000 * 1000 /* 1 min */); + ~ObTenantFreezeGuard(); +private: + common::ObMemstoreAllocatorMgr *allocator_mgr_; + int64_t pre_retire_pos_; + int &error_code_; + ObTimeGuard time_guard_; +}; + } // storage } // oceanbase #endif