hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

FREEZE_CHECKER: check whether the tenant memstore is really frozen

Browse Source
This commit is contained in:
obdev 2023-05-09 18:43:01 +00:00 committed by ob-robot
parent 186be18745
commit 28a864535f
8 changed files with 140 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/observer/virtual_table/ob_all_virtual_tenant_memstore_allocator_info.cpp
View File

@ -37,7 +37,7 @@ public:
memtable::ObMemtable& mt = handle->mt_;
ObLSID ls_id;
item.protection_clock_ = handle->get_protection_clock();
item.is_active_ = mt.is_active_memtable();
item.is_active_ = handle->is_active();
item.ls_id_ = (OB_SUCCESS == mt.get_ls_id(ls_id)) ? ls_id.id() : ObLSID::INVALID_LS_ID;
item.tablet_id_ = mt.get_key().tablet_id_.id();
item.scn_range_ = mt.get_scn_range();

16
src/share/allocator/ob_gmemstore_allocator.cpp
View File

@ -30,7 +30,21 @@ int FrozenMemstoreInfoLogger::operator()(ObDLink* link)
int ret = OB_SUCCESS;
ObGMemstoreAllocator::AllocHandle* handle = CONTAINER_OF(link, typeof(*handle), total_list_);
memtable::ObMemtable& mt = handle->mt_;
if (mt.is_frozen_memtable()) {
if (handle->is_frozen()) {
if (OB_FAIL(databuff_print_obj(buf_, limit_, pos_, mt))) {
} else {
ret = databuff_printf(buf_, limit_, pos_, ",");
}
}
return ret;
}
int ActiveMemstoreInfoLogger::operator()(ObDLink* link)
{
int ret = OB_SUCCESS;
ObGMemstoreAllocator::AllocHandle* handle = CONTAINER_OF(link, typeof(*handle), total_list_);
memtable::ObMemtable& mt = handle->mt_;
if (handle->is_active()) {
if (OB_FAIL(databuff_print_obj(buf_, limit_, pos_, mt))) {
} else {
ret = databuff_printf(buf_, limit_, pos_, ",");

28
src/share/allocator/ob_gmemstore_allocator.h
View File

@ -33,6 +33,17 @@ struct FrozenMemstoreInfoLogger
int64_t limit_;
int64_t pos_;
};
struct ActiveMemstoreInfoLogger
{
ActiveMemstoreInfoLogger(char* buf, int64_t limit): buf_(buf), limit_(limit), pos_(0) {}
~ActiveMemstoreInfoLogger() {}
int operator()(ObDLink* link);
char* buf_;
int64_t limit_;
int64_t pos_;
};
class ObGMemstoreAllocator
{
public:
@ -120,11 +131,11 @@ public:
void* alloc(AllocHandle& handle, int64_t size);
void set_frozen(AllocHandle& handle);
template<typename Func>
int for_each(Func& f) {
int for_each(Func& f, const bool reverse=false) {
int ret = common::OB_SUCCESS;
ObDLink* iter = NULL;
LockGuard guard(lock_);
while(OB_SUCC(ret) && NULL != (iter = hlist_.next(iter))) {
while(OB_SUCC(ret) && NULL != (iter = (reverse ? hlist_.prev(iter) : hlist_.next(iter)))) {
ret = f(iter);
}
return ret;
@ -134,12 +145,23 @@ public:
int64_t hazard = hlist_.hazard();
return hazard == INT64_MAX? 0: (arena_.allocated() - hazard);
}
int64_t get_frozen_memstore_pos() const {
int64_t hazard = hlist_.hazard();
return hazard == INT64_MAX? 0: hazard;
}
int64_t get_mem_total_memstore_used() const { return arena_.hold(); }
void log_frozen_memstore_info(char* buf, int64_t limit) {
if (NULL != buf && limit > 0) {
FrozenMemstoreInfoLogger logger(buf, limit);
buf[0] = 0;
(void)for_each(logger);
(void)for_each(logger, true /* reverse */);
}
}
void log_active_memstore_info(char *buf, int64_t limit) {
if (NULL != buf && limit > 0) {
ActiveMemstoreInfoLogger logger(buf, limit);
buf[0] = 0;
(void)for_each(logger, true /* reverse */);
}
}
public:

11
src/share/allocator/ob_handle_list.h
View File

@ -45,6 +45,16 @@ public:
}
return iter;
}
DLink* prev(DLink* iter) {
if (NULL == iter) {
iter = &tail_;
}
iter = (DLink*)iter->prev_;
if (&head_ == iter) {
iter = NULL;
}
return iter;
}
static void dlink_insert(ObDLink* cur, ObDLink* x)
{
ObDLink* next = (ObDLink*)cur->next_;
@ -129,6 +139,7 @@ public:
bool is_empty() const { return ATOMIC_LOAD(&total_count_) <= 0; }
int64_t hazard() const { return ATOMIC_LOAD(&hazard_); }
DLink* next(DLink* iter) { return total_list_.next(iter); }
DLink* prev(DLink* iter) { return total_list_.prev(iter); }
protected:
void set_frozen_(Handle& handle);
int64_t alloc_id() { return ATOMIC_AAF(&id_, 1); }

3
src/storage/tx_storage/ob_tenant_freezer.cpp
View File

@ -316,6 +316,7 @@ int ObTenantFreezer::tenant_freeze_()
ObLSService *ls_srv = MTL(ObLSService *);
FLOG_INFO("[TenantFreezer] tenant_freeze start", KR(ret));
ObTenantFreezeGuard freeze_guard(allocator_mgr_, ret);
if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::TXSTORAGE_MOD))) {
LOG_WARN("[TenantFreezer] fail to get log stream iterator", KR(ret));
} else {
@ -1362,6 +1363,8 @@ int ObTenantFreezer::do_major_if_need_(const bool need_freeze)
// update frozen scn
} else if (!need_freeze) {
// no need major
} else if (!is_major_freeze_turn_()) {
// do nothing
} else if (OB_FAIL(get_global_frozen_scn_(frozen_scn))) {
LOG_WARN("fail to get global frozen version", K(ret));
} else if (0 != frozen_scn && OB_FAIL(tenant_info_.update_frozen_scn(frozen_scn))) {

2
src/storage/tx_storage/ob_tenant_freezer.h
View File

@ -153,7 +153,7 @@ private:
const int64_t abs_timeout_ts);
// freeze all the ls of this tenant.
// return the first failed code.
static int tenant_freeze_();
int tenant_freeze_();
// we can only deal with freeze one by one.
// set tenant freezing will prevent a new freeze.
int set_tenant_freezing_();

66
src/storage/tx_storage/ob_tenant_freezer_common.cpp
View File

@ -10,7 +10,14 @@
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "lib/oblog/ob_log.h"
#include "lib/alloc/alloc_func.h"
#include "share/allocator/ob_gmemstore_allocator.h"
#include "share/allocator/ob_memstore_allocator_mgr.h"
#include "share/ob_force_print_log.h"
#include "share/rc/ob_tenant_base.h"
#include "storage/tx_storage/ob_tenant_freezer_common.h"
namespace oceanbase
@ -18,6 +25,7 @@ namespace oceanbase
using namespace lib;
namespace storage
{
typedef ObMemstoreAllocatorMgr::TAllocator ObTenantMemstoreAllocator;
DEF_TO_STRING(ObTenantFreezeArg)
{
@ -144,5 +152,63 @@ void ObTenantInfo::get_freeze_ctx(ObTenantFreezeCtx &ctx) const
ctx.mem_memstore_limit_ = mem_memstore_limit_;
}
ObTenantFreezeGuard::ObTenantFreezeGuard(common::ObMemstoreAllocatorMgr *allocator_mgr,
int &err_code,
const int64_t warn_threshold)
: allocator_mgr_(nullptr),
pre_retire_pos_(0),
error_code_(err_code),
time_guard_("FREEZE_CHECKER", warn_threshold)
{
int ret = OB_SUCCESS;
ObTenantMemstoreAllocator *tenant_allocator = NULL;
const uint64_t tenant_id = MTL_ID();
if (OB_NOT_NULL(allocator_mgr)) {
allocator_mgr_ = allocator_mgr;
if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id,
tenant_allocator))) {
LOG_WARN("[FREEZE_CHECKER] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id));
} else if (NULL == tenant_allocator) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("[FREEZE_CHECKER] tenant memstore allocator is NULL", KR(ret), K(tenant_id));
} else {
pre_retire_pos_ = tenant_allocator->get_retire_clock();
}
}
}
ObTenantFreezeGuard::~ObTenantFreezeGuard()
{
int ret = OB_SUCCESS;
ObTenantMemstoreAllocator *tenant_allocator = NULL;
const uint64_t tenant_id = MTL_ID();
int64_t curr_frozen_pos = 0;
if (OB_ISNULL(allocator_mgr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("[FREEZE_CHECKER]freeze guard invalid", KR(ret), K_(allocator_mgr), K(lbt()));
} else if (OB_FAIL(error_code_)) {
LOG_WARN("[FREEZE_CHECKER]tenant freeze failed, skip check frozen memstore", KR(ret));
} else if (OB_FAIL(allocator_mgr_->get_tenant_memstore_allocator(tenant_id,
tenant_allocator))) {
LOG_WARN("[FREEZE_CHECKER] failed to get_tenant_memstore_allocator", KR(ret), K(tenant_id));
} else if (NULL == tenant_allocator) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("[FREEZE_CHECKER] tenant memstore allocator is NULL", KR(ret), K(tenant_id));
} else {
curr_frozen_pos = tenant_allocator->get_frozen_memstore_pos();
const bool retired_mem_frozen = (curr_frozen_pos >= pre_retire_pos_);
const bool has_no_active_memtable = (curr_frozen_pos == 0);
if (!(retired_mem_frozen || has_no_active_memtable)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("[FREEZE_CHECKER]there may be frequent tenant freeze", KR(ret), K(curr_frozen_pos),
K_(pre_retire_pos), K(retired_mem_frozen), K(has_no_active_memtable));
char active_mt_info[DEFAULT_BUF_LENGTH];
tenant_allocator->log_active_memstore_info(active_mt_info,
sizeof(active_mt_info));
FLOG_INFO("[FREEZE_CHECKER] oldest active memtable", "list", active_mt_info);
}
}
}
} // storage
} // oceanbase

18
src/storage/tx_storage/ob_tenant_freezer_common.h
View File

@ -18,6 +18,10 @@
#define OCEABASE_STORAGE_TENANT_FREEZER_COMMON_
namespace oceanbase
{
namespace common
{
class ObMemstoreAllocatorMgr;
}
namespace storage
{
struct ObTenantFreezeArg
@ -116,6 +120,20 @@ private:
DISALLOW_COPY_AND_ASSIGN(ObTenantInfo);
};
class ObTenantFreezeGuard
{
public:
ObTenantFreezeGuard(common::ObMemstoreAllocatorMgr *allocator_mgr,
int &ret,
const int64_t warn_threshold = 60 * 1000 * 1000 /* 1 min */);
~ObTenantFreezeGuard();
private:
common::ObMemstoreAllocatorMgr *allocator_mgr_;
int64_t pre_retire_pos_;
int &error_code_;
ObTimeGuard time_guard_;
};
} // storage
} // oceanbase
#endif