From 1ded3b511ccdf6d64cf2645f8beb0dcd628604da Mon Sep 17 00:00:00 2001 From: ZenoWang Date: Tue, 27 Feb 2024 04:46:00 +0000 Subject: [PATCH] [FIX] Block replay when logstream freeze --- src/storage/ls/ob_freezer.cpp | 90 +++++++++++++++++++- src/storage/ls/ob_freezer.h | 54 +++--------- src/storage/tx_storage/ob_tenant_freezer.cpp | 3 + 3 files changed, 104 insertions(+), 43 deletions(-) diff --git a/src/storage/ls/ob_freezer.cpp b/src/storage/ls/ob_freezer.cpp index 866708443c..403f3579a6 100644 --- a/src/storage/ls/ob_freezer.cpp +++ b/src/storage/ls/ob_freezer.cpp @@ -378,6 +378,8 @@ ObFreezer::ObFreezer() empty_memtable_cnt_(0), high_priority_freeze_cnt_(0), low_priority_freeze_cnt_(0), + pend_replay_cnt_(0), + byte_lock_(), need_resubmit_log_(false), enable_(false), is_inited_(false) @@ -392,6 +394,8 @@ ObFreezer::ObFreezer(ObLS *ls) empty_memtable_cnt_(0), high_priority_freeze_cnt_(0), low_priority_freeze_cnt_(0), + pend_replay_cnt_(0), + byte_lock_(), need_resubmit_log_(false), enable_(false), is_inited_(false) @@ -412,6 +416,7 @@ void ObFreezer::reset() empty_memtable_cnt_ = 0; high_priority_freeze_cnt_ = 0; low_priority_freeze_cnt_ = 0; + pend_replay_cnt_ = 0; need_resubmit_log_ = false; enable_ = false; is_inited_ = false; @@ -433,6 +438,7 @@ int ObFreezer::init(ObLS *ls) empty_memtable_cnt_ = 0; high_priority_freeze_cnt_ = 0; low_priority_freeze_cnt_ = 0; + pend_replay_cnt_ = 0; need_resubmit_log_ = false; is_inited_ = true; @@ -570,6 +576,7 @@ int ObFreezer::ls_freeze_task() const int64_t start = ObTimeUtility::current_time(); int64_t last_submit_log_time = start; uint32_t freeze_clock = get_freeze_clock(); + PendTenantReplayGuard pend_replay_guard; TRANS_LOG(INFO, "[Freezer] freeze_clock", K(ls_id), K(freeze_clock)); // wait till all memtables are moved from frozen_list to prepare_list @@ -1832,12 +1839,57 @@ void ObFreezer::set_ls_freeze_end_() ATOMIC_DEC(&high_priority_freeze_cnt_); } +int ObFreezer::pend_ls_replay() +{ + int ret = OB_SUCCESS; + common::ObByteLockGuard guard(byte_lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), K(get_ls_id()), KP(this)); + } else if (pend_replay_cnt_ < 0) { + TRANS_LOG(ERROR, "[Freezer] invalid pend_replay_cnt", KR(ret), K(get_ls_id()), KP(this), K(pend_replay_cnt_)); + } else if (FALSE_IT(pend_replay_cnt_++)) { + } else if (1 == pend_replay_cnt_) { + if (OB_FAIL(get_ls_log_handler()->pend_submit_replay_log())) { + pend_replay_cnt_--; + TRANS_LOG(WARN, "[Freezer] pend ls replay failed", KR(ret), K(get_ls_id()), KP(this)); + } + } + return ret; +} + +int ObFreezer::restore_ls_replay() +{ + int ret = OB_SUCCESS; + common::ObByteLockGuard guard(byte_lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "[Freezer] not inited", KR(ret), K(get_ls_id()), KP(this)); + } else if (0 >= pend_replay_cnt_) { + TRANS_LOG(INFO, + "[Freezer] no need restore replay cause pend replay may failed", + KR(ret), + K(get_ls_id()), + KP(this), + K(pend_replay_cnt_)); + } else if (FALSE_IT(pend_replay_cnt_--)) { + } else if (0 == pend_replay_cnt_) { + if (OB_FAIL(get_ls_log_handler()->restore_submit_replay_log())) { + if (get_ls_log_handler()->is_valid()) { + TRANS_LOG(ERROR, "restore replay failed. please check if logstream is removed", KR(ret), K(get_ls_id())); + } else { + TRANS_LOG(WARN, "restore replay failed. ls log handler is invalid", KR(ret), K(get_ls_id())); + } + } + } + return ret; +} + ObFreezer::ObLSFreezeGuard::ObLSFreezeGuard(ObFreezer &parent) : parent_(parent) { parent_.set_ls_freeze_begin_(); } - ObFreezer::ObLSFreezeGuard::~ObLSFreezeGuard() { parent_.set_ls_freeze_end_(); @@ -1872,5 +1924,41 @@ int ObFreezer::ObTabletFreezeGuard::try_set_tablet_freeze_begin() return ret; } +ObFreezer::PendTenantReplayGuard::PendTenantReplayGuard() +{ + int ret = OB_SUCCESS; + common::ObSharedGuard iter; + ObLSService *ls_srv = MTL(ObLSService *); + if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "[ObFreezer] fail to get ls iterator", KR(ret)); + } else { + ObLS *ls = nullptr; + while (OB_SUCC(iter->get_next(ls))) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(ls->get_freezer()->pend_ls_replay())) { + STORAGE_LOG(WARN, "[ObFreezer] pend replay failed", KR(ret), KPC(ls)); + } + } + } +} + +ObFreezer::PendTenantReplayGuard::~PendTenantReplayGuard() +{ + int ret = OB_SUCCESS; + common::ObSharedGuard iter; + ObLSService *ls_srv = MTL(ObLSService *); + if (OB_FAIL(ls_srv->get_ls_iter(iter, ObLSGetMod::STORAGE_MOD))) { + STORAGE_LOG(WARN, "[ObFreezer] fail to get ls iterator", KR(ret)); + } else { + ObLS *ls = nullptr; + while (OB_SUCC(iter->get_next(ls))) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(ls->get_freezer()->restore_ls_replay())) { + STORAGE_LOG(WARN, "[ObFreezer] restore replay failed", KR(ret), KPC(ls)); + } + } + } +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/ls/ob_freezer.h b/src/storage/ls/ob_freezer.h index ca7fcfb085..8292dc6544 100644 --- a/src/storage/ls/ob_freezer.h +++ b/src/storage/ls/ob_freezer.h @@ -46,53 +46,12 @@ class ObLSTabletService; class ObTablet; class ObLSWRSHandler; class ObTableHandleV2; +class ObLSIterator; namespace checkpoint { class ObDataCheckpoint; } -class ObReplaySubmitLogPendingWhenFreezeGuard -{ -public: - ObReplaySubmitLogPendingWhenFreezeGuard(logservice::ObILogHandler *loghandler, - share::ObLSID &ls_id) - : is_follower_(false), - loghandler_(loghandler), - ls_id_(ls_id) { - int ret = OB_SUCCESS; - int64_t proposal_id = 0; - common::ObRole ls_role = common::ObRole::INVALID_ROLE; - if (OB_FAIL(loghandler_->get_role(ls_role, proposal_id))) { - STORAGE_LOG(WARN, "get ls role fail", K(ret), K(ls_id_)); - } else if (common::ObRole::FOLLOWER == ls_role) { - is_follower_ = true; - } - - if (is_follower_) { - if (OB_ISNULL(loghandler_)) { - STORAGE_LOG(WARN, "loghandler should not null", K(ls_id_)); - } else if (OB_FAIL(loghandler_->pend_submit_replay_log())) { - STORAGE_LOG(ERROR, "pend_submit_replay_log failed", K(ls_id_)); - } - } - } - - ~ObReplaySubmitLogPendingWhenFreezeGuard() { - int ret = OB_SUCCESS; - if (is_follower_) { - if (OB_ISNULL(loghandler_)) { - STORAGE_LOG(WARN, "loghandler should not null", K(ls_id_)); - } else if (OB_FAIL(loghandler_->restore_submit_replay_log())) { - STORAGE_LOG(ERROR, "restore_submit_replay_log failed", K(ls_id_)); - } - } - } - -private: - bool is_follower_; - logservice::ObILogHandler *loghandler_; - share::ObLSID ls_id_; -}; class ObFreezeState { @@ -267,6 +226,8 @@ public: void set_need_resubmit_log(bool flag) { return ATOMIC_STORE(&need_resubmit_log_, flag); } // only used after start freeze_task successfully int wait_freeze_finished(ObFuture &result); + int pend_ls_replay(); + int restore_ls_replay(); private: class ObLSFreezeGuard @@ -287,6 +248,12 @@ private: bool need_release_; ObFreezer &parent_; }; + class PendTenantReplayGuard + { + public: + PendTenantReplayGuard(); + ~PendTenantReplayGuard(); + }; private: /* freeze_flag */ int set_freeze_flag(); @@ -341,6 +308,9 @@ private: // make sure ls freeze has higher priority than tablet freeze int64_t high_priority_freeze_cnt_; // waiting and freeze cnt int64_t low_priority_freeze_cnt_; // freeze tablet cnt + int64_t pend_replay_cnt_; + common::ObByteLock byte_lock_; // only used to control pend_replay_cnt_ + bool need_resubmit_log_; bool enable_; // whether we can do freeze now diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index 916c2f63c2..fad5470406 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -498,6 +498,9 @@ int ObTenantFreezer::check_and_freeze_normal_data_(ObTenantFreezeCtx &ctx) if (OB_TMP_FAIL(do_minor_freeze_(ctx))) { LOG_WARN("[TenantFreezer] fail to do minor freeze", K(tmp_ret)); } + if (OB_TMP_FAIL(post_tx_data_freeze_request_())) { + LOG_WARN("[TenantFreezer] fail to do tx data self freeze", KR(tmp_ret)); + } } } return ret;