hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FIX] Block replay when logstream freeze

Browse Source
This commit is contained in:
ZenoWang
2024-02-27 04:46:00 +00:00
committed by ob-robot
parent f91e753574
commit 1ded3b511c
3 changed files with 104 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
src/storage/ls/ob_freezer.cpp
View File

@ -378,6 +378,8 @@ ObFreezer::ObFreezer()
empty_memtable_cnt_(0),
high_priority_freeze_cnt_(0),
low_priority_freeze_cnt_(0),
pend_replay_cnt_(0),
byte_lock_(),
need_resubmit_log_(false),
enable_(false),
is_inited_(false)
@ -392,6 +394,8 @@ ObFreezer::ObFreezer(ObLS *ls)
empty_memtable_cnt_(0),
high_priority_freeze_cnt_(0),
low_priority_freeze_cnt_(0),
pend_replay_cnt_(0),
byte_lock_(),
need_resubmit_log_(false),
enable_(false),
is_inited_(false)
@ -412,6 +416,7 @@ void ObFreezer::reset()
empty_memtable_cnt_ = 0;
high_priority_freeze_cnt_ = 0;
low_priority_freeze_cnt_ = 0;
pend_replay_cnt_ = 0;
need_resubmit_log_ = false;
enable_ = false;
is_inited_ = false;
@ -433,6 +438,7 @@ int ObFreezer::init(ObLS *ls)
empty_memtable_cnt_ = 0;
high_priority_freeze_cnt_ = 0;
low_priority_freeze_cnt_ = 0;
pend_replay_cnt_ = 0;
need_resubmit_log_ = false;
is_inited_ = true;
@ -570,6 +576,7 @@ int ObFreezer::ls_freeze_task()
const int64_t start = ObTimeUtility::current_time();
int64_t last_submit_log_time = start;
uint32_t freeze_clock = get_freeze_clock();
PendTenantReplayGuard pend_replay_guard;
TRANS_LOG(INFO, "[Freezer] freeze_clock", K(ls_id), K(freeze_clock));
// wait till all memtables are moved from frozen_list to prepare_list
@ -1832,12 +1839,57 @@ void ObFreezer::set_ls_freeze_end_()
ATOMIC_DEC(&high_priority_freeze_cnt_);
}
int ObFreezer::pend_ls_replay()
{
int ret = OB_SUCCESS;
common::ObByteLockGuard guard(byte_lock_);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), K(get_ls_id()), KP(this));
} else if (pend_replay_cnt_ < 0) {
TRANS_LOG(ERROR, "[Freezer] invalid pend_replay_cnt", KR(ret), K(get_ls_id()), KP(this), K(pend_replay_cnt_));
} else if (FALSE_IT(pend_replay_cnt_++)) {
} else if (1 == pend_replay_cnt_) {
if (OB_FAIL(get_ls_log_handler()->pend_submit_replay_log())) {
pend_replay_cnt_--;
TRANS_LOG(WARN, "[Freezer] pend ls replay failed", KR(ret), K(get_ls_id()), KP(this));
}
}
return ret;
}
int ObFreezer::restore_ls_replay()
{
int ret = OB_SUCCESS;
common::ObByteLockGuard guard(byte_lock_);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), K(get_ls_id()), KP(this));
} else if (0 >= pend_replay_cnt_) {
TRANS_LOG(INFO,
"[Freezer] no need restore replay cause pend replay may failed",
KR(ret),
K(get_ls_id()),
KP(this),
K(pend_replay_cnt_));
} else if (FALSE_IT(pend_replay_cnt_--)) {
} else if (0 == pend_replay_cnt_) {
if (OB_FAIL(get_ls_log_handler()->restore_submit_replay_log())) {
if (get_ls_log_handler()->is_valid()) {
TRANS_LOG(ERROR, "restore replay failed. please check if logstream is removed", KR(ret), K(get_ls_id()));
} else {
TRANS_LOG(WARN, "restore replay failed. ls log handler is invalid", KR(ret), K(get_ls_id()));
}
}
}
return ret;
}
ObFreezer::ObLSFreezeGuard::ObLSFreezeGuard(ObFreezer &parent)
: parent_(parent)
{
parent_.set_ls_freeze_begin_();
}
ObFreezer::ObLSFreezeGuard::~ObLSFreezeGuard()
{
parent_.set_ls_freeze_end_();
@ -1872,5 +1924,41 @@ int ObFreezer::ObTabletFreezeGuard::try_set_tablet_freeze_begin()
return ret;
}
ObFreezer::PendTenantReplayGuard::PendTenantReplayGuard()
{
int ret = OB_SUCCESS;
common::ObSharedGuard<ObLSIterator> iter;
ObLSService *ls_srv = MTL(ObLSService *);
if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "[ObFreezer] fail to get ls iterator", KR(ret));
} else {
ObLS *ls = nullptr;
while (OB_SUCC(iter->get_next(ls))) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(ls->get_freezer()->pend_ls_replay())) {
STORAGE_LOG(WARN, "[ObFreezer] pend replay failed", KR(ret), KPC(ls));
}
}
}
}
ObFreezer::PendTenantReplayGuard::~PendTenantReplayGuard()
{
int ret = OB_SUCCESS;
common::ObSharedGuard<ObLSIterator> iter;
ObLSService *ls_srv = MTL(ObLSService *);
if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::STORAGE_MOD))) {
STORAGE_LOG(WARN, "[ObFreezer] fail to get ls iterator", KR(ret));
} else {
ObLS *ls = nullptr;
while (OB_SUCC(iter->get_next(ls))) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(ls->get_freezer()->restore_ls_replay())) {
STORAGE_LOG(WARN, "[ObFreezer] restore replay failed", KR(ret), KPC(ls));
}
}
}
}
} // namespace storage
} // namespace oceanbase

54
src/storage/ls/ob_freezer.h
View File

@ -46,53 +46,12 @@ class ObLSTabletService;
class ObTablet;
class ObLSWRSHandler;
class ObTableHandleV2;
class ObLSIterator;
namespace checkpoint
{
class ObDataCheckpoint;
}
class ObReplaySubmitLogPendingWhenFreezeGuard
{
public:
ObReplaySubmitLogPendingWhenFreezeGuard(logservice::ObILogHandler *loghandler,
share::ObLSID &ls_id)
: is_follower_(false),
loghandler_(loghandler),
ls_id_(ls_id) {
int ret = OB_SUCCESS;
int64_t proposal_id = 0;
common::ObRole ls_role = common::ObRole::INVALID_ROLE;
if (OB_FAIL(loghandler_->get_role(ls_role, proposal_id))) {
STORAGE_LOG(WARN, "get ls role fail", K(ret), K(ls_id_));
} else if (common::ObRole::FOLLOWER == ls_role) {
is_follower_ = true;
}
if (is_follower_) {
if (OB_ISNULL(loghandler_)) {
STORAGE_LOG(WARN, "loghandler should not null", K(ls_id_));
} else if (OB_FAIL(loghandler_->pend_submit_replay_log())) {
STORAGE_LOG(ERROR, "pend_submit_replay_log failed", K(ls_id_));
}
}
}
~ObReplaySubmitLogPendingWhenFreezeGuard() {
int ret = OB_SUCCESS;
if (is_follower_) {
if (OB_ISNULL(loghandler_)) {
STORAGE_LOG(WARN, "loghandler should not null", K(ls_id_));
} else if (OB_FAIL(loghandler_->restore_submit_replay_log())) {
STORAGE_LOG(ERROR, "restore_submit_replay_log failed", K(ls_id_));
}
}
}
private:
bool is_follower_;
logservice::ObILogHandler *loghandler_;
share::ObLSID ls_id_;
};
class ObFreezeState
{
@ -267,6 +226,8 @@ public:
void set_need_resubmit_log(bool flag) { return ATOMIC_STORE(&need_resubmit_log_, flag); }
// only used after start freeze_task successfully
int wait_freeze_finished(ObFuture<int> &result);
int pend_ls_replay();
int restore_ls_replay();
private:
class ObLSFreezeGuard
@ -287,6 +248,12 @@ private:
bool need_release_;
ObFreezer &parent_;
};
class PendTenantReplayGuard
{
public:
PendTenantReplayGuard();
~PendTenantReplayGuard();
};
private:
/* freeze_flag */
int set_freeze_flag();
@ -341,6 +308,9 @@ private:
// make sure ls freeze has higher priority than tablet freeze
int64_t high_priority_freeze_cnt_; // waiting and freeze cnt
int64_t low_priority_freeze_cnt_; // freeze tablet cnt
int64_t pend_replay_cnt_;
common::ObByteLock byte_lock_; // only used to control pend_replay_cnt_
bool need_resubmit_log_;
bool enable_; // whether we can do freeze now

3
src/storage/tx_storage/ob_tenant_freezer.cpp
View File

@ -498,6 +498,9 @@ int ObTenantFreezer::check_and_freeze_normal_data_(ObTenantFreezeCtx &ctx)
if (OB_TMP_FAIL(do_minor_freeze_(ctx))) {
LOG_WARN("[TenantFreezer] fail to do minor freeze", K(tmp_ret));
}
if (OB_TMP_FAIL(post_tx_data_freeze_request_())) {
LOG_WARN("[TenantFreezer] fail to do tx data self freeze", KR(tmp_ret));
}
}
}
return ret;