hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

clear all memory state in offline for dup_table

Browse Source
This commit is contained in:
KyrielightWei
2023-06-13 03:18:05 +00:00
committed by ob-robot
parent 0c27bf4af7
commit 20fe056fc2
13 changed files with 773 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/storage/high_availability/ob_ls_migration.cpp
View File

@ -1253,7 +1253,8 @@ int ObStartMigrationTask::update_ls_()
if (OB_FAIL(ls->update_ls_meta(update_restore_status,
ctx_->src_ls_meta_package_.ls_meta_))) {
LOG_WARN("failed to update ls meta", K(ret), KPC(ctx_));
} else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_))) {
} else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_,
true /*need_flush_slog*/))) {
LOG_WARN("failed to set dup table ls meta", K(ret), KPC(ctx_));
} else if (OB_FAIL(ls->get_end_lsn(end_lsn))) {
LOG_WARN("failed to get end lsn", K(ret), KPC(ctx_));

3
src/storage/high_availability/ob_ls_restore.cpp
View File

@ -1083,7 +1083,8 @@ int ObStartLSRestoreTask::update_ls_meta_()
} else if (OB_FAIL(ls->update_ls_meta(false/*don't update restore status*/,
ctx_->src_ls_meta_package_.ls_meta_))) {
LOG_WARN("fail to update ls meta", K(ret), KPC(ls), KPC(ctx_));
} else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_))) {
} else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_,
true /*need_flush_slog*/))) {
LOG_WARN("fail to set dup table ls meta", K(ret), KPC(ctx_));
} else {
LOG_INFO("update ls meta succeed", KPC(ls), KPC(ctx_));

11
src/storage/ls/ob_ls.cpp
View File

@ -465,6 +465,7 @@ int ObLS::start()
LOG_WARN("set max replay commit scn fail", K(ret), K(ls_meta_.get_clog_checkpoint_scn()));
} else {
checkpoint_executor_.start();
dup_table_ls_handler_.start();
LOG_INFO("start_ls finish", KR(ret), KPC(this));
// do nothing
}
@ -494,6 +495,7 @@ int ObLS::stop_()
tx_table_.stop();
ls_restore_handler_.stop();
keep_alive_ls_handler_.stop();
dup_table_ls_handler_.stop();
log_handler_.reset_election_priority();
restore_handler_.stop();
if (OB_FAIL(log_handler_.stop())) {
@ -579,6 +581,7 @@ bool ObLS::safe_to_destroy()
bool is_ls_restore_handler_safe = false;
bool is_tablet_service_safe = false;
bool is_data_check_point_safe = false;
bool is_dup_table_handler_safe = false;
bool is_log_handler_safe = false;
if (OB_FAIL(ls_tablet_svr_.safe_to_destroy(is_tablet_service_safe))) {
@ -587,6 +590,9 @@ bool ObLS::safe_to_destroy()
} else if (OB_FAIL(data_checkpoint_.safe_to_destroy(is_data_check_point_safe))) {
LOG_WARN("data_checkpoint check safe to destroy failed", K(ret), KPC(this));
} else if (!is_data_check_point_safe) {
} else if (OB_FAIL(dup_table_ls_handler_.safe_to_destroy(is_dup_table_handler_safe))) {
LOG_WARN("dup table ls handler safe to destroy failed", K(ret), KPC(this));
} else if (!is_dup_table_handler_safe) {
} else if (OB_FAIL(ls_restore_handler_.safe_to_destroy(is_ls_restore_handler_safe))) {
LOG_WARN("ls restore handler safe to destroy failed", K(ret), KPC(this));
} else if (!is_ls_restore_handler_safe) {
@ -605,6 +611,7 @@ bool ObLS::safe_to_destroy()
if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) {
LOG_WARN("this ls is not safe to destroy", K(is_safe),
K(is_tablet_service_safe), K(is_data_check_point_safe),
K(is_dup_table_handler_safe),
K(is_ls_restore_handler_safe), K(is_log_handler_safe),
"ls_ref", ref_mgr_.get_total_ref_cnt(),
K(ret), KP(this), KPC(this));
@ -782,6 +789,8 @@ int ObLS::offline_()
LOG_WARN("ddl log handler offline failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(offline_tx_())) {
LOG_WARN("offline tx service failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(dup_table_ls_handler_.offline())) {
LOG_WARN("offline dup table ls handler failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(lock_table_.offline())) {
LOG_WARN("lock table offline failed", K(ret), K(ls_meta_));
// force release memtables created by force_tablet_freeze called during major
@ -881,6 +890,8 @@ int ObLS::online()
LOG_WARN("tablet service online failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(lock_table_.online())) {
LOG_WARN("lock table online failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(dup_table_ls_handler_.online())) {
LOG_WARN("dup table ls handler online failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(online_tx_())) {
LOG_WARN("ls tx online failed", K(ret), K(ls_meta_));
} else if (OB_FAIL(ls_ddl_log_handler_.online())) {

186
src/storage/tx/ob_dup_table_base.cpp
View File

@ -38,6 +38,164 @@ const int64_t DupTableDiagStd::DUP_DIAG_PRINT_INTERVAL[DupTableDiagStd::TypeInde
3 * 60 * 1000 * 1000 // 3min , ts_sync_print_interval
};
/*******************************************************
* Dup_Table LS Role State
*******************************************************/
int64_t *ObDupTableLSRoleStateContainer::get_target_state_ref(ObDupTableLSRoleState target_state)
{
int64_t *state_member = nullptr;
switch (target_state) {
case ObDupTableLSRoleState::LS_REVOKE_SUCC:
case ObDupTableLSRoleState::LS_TAKEOVER_SUCC: {
state_member = &role_state_;
break;
}
case ObDupTableLSRoleState::LS_OFFLINE_SUCC:
case ObDupTableLSRoleState::LS_ONLINE_SUCC: {
state_member = &offline_state_;
break;
}
case ObDupTableLSRoleState::LS_START_SUCC:
case ObDupTableLSRoleState::LS_STOP_SUCC: {
state_member = &stop_state_;
break;
}
default: {
state_member = nullptr;
DUP_TABLE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "undefined target role state", K(target_state),
KPC(this));
break;
}
}
return state_member;
}
bool ObDupTableLSRoleStateContainer::check_target_state(ObDupTableLSRoleState target_state)
{
bool is_same = false;
int64_t *state_member = get_target_state_ref(target_state);
if (OB_NOT_NULL(state_member)) {
ObDupTableLSRoleState cur_state = static_cast<ObDupTableLSRoleState>(ATOMIC_LOAD(state_member));
if (target_state == cur_state) {
is_same = true;
} else if (ObDupTableLSRoleState::ROLE_STATE_CHANGING == cur_state) {
is_same = false;
DUP_TABLE_LOG(INFO, "The Role State is Changing", K(is_same), K(cur_state), KPC(this));
} else {
is_same = false;
}
}
return is_same;
}
bool ObDupTableLSRoleStateHelper::is_leader()
{
return cur_state_.check_target_state(ObDupTableLSRoleState::LS_TAKEOVER_SUCC);
}
bool ObDupTableLSRoleStateHelper::is_follower()
{
return cur_state_.check_target_state(ObDupTableLSRoleState::LS_REVOKE_SUCC);
}
bool ObDupTableLSRoleStateHelper::is_offline()
{
return cur_state_.check_target_state(ObDupTableLSRoleState::LS_OFFLINE_SUCC);
}
bool ObDupTableLSRoleStateHelper::is_online()
{
return cur_state_.check_target_state(ObDupTableLSRoleState::LS_ONLINE_SUCC);
}
bool ObDupTableLSRoleStateHelper::is_stopped()
{
return cur_state_.check_target_state(ObDupTableLSRoleState::LS_STOP_SUCC);
}
bool ObDupTableLSRoleStateHelper::is_started()
{
return cur_state_.check_target_state(ObDupTableLSRoleState::LS_START_SUCC);
}
#define DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state) \
int64_t *state_member = cur_state_.get_target_state_ref(target_state); \
int64_t *backup_state_member = restore_state.get_target_state_ref(target_state); \
ObDupTableLSRoleState cur_ls_state = ObDupTableLSRoleState::UNKNOWN; \
if (OB_ISNULL(state_member) || OB_ISNULL(backup_state_member)) { \
ret = OB_INVALID_ARGUMENT; \
DUP_TABLE_LOG(WARN, "invalid arguments", K(ret), KP(state_member), KP(backup_state_member), \
K(target_state), KPC(this)); \
} else if (OB_FALSE_IT(cur_ls_state = \
static_cast<ObDupTableLSRoleState>(ATOMIC_LOAD(state_member)))) { \
}
int ObDupTableLSRoleStateHelper::prepare_state_change(const ObDupTableLSRoleState &target_state,
ObDupTableLSRoleStateContainer &restore_state)
{
int ret = OB_SUCCESS;
DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)
if (OB_FAIL(ret)) {
// do nothing
} else if (target_state == cur_ls_state) {
ret = OB_NO_NEED_UPDATE;
DUP_TABLE_LOG(INFO, "the cur state has already been same as target_state", K(ret),
K(target_state), KPC(this));
} else if (cur_ls_state == ObDupTableLSRoleState::UNKNOWN) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(INFO, "invalid role state", K(cur_ls_state), KPC(this));
} else {
if (ObDupTableLSRoleState::ROLE_STATE_CHANGING == cur_ls_state) {
DUP_TABLE_LOG(INFO,
"cur_ls_state is the ROLE_STATE_CHANGING, may be error in the last role change",
K(ret), K(cur_ls_state), K(target_state), KPC(this));
}
*backup_state_member = static_cast<int64_t>(cur_ls_state);
ATOMIC_STORE(state_member, static_cast<int64_t>(ObDupTableLSRoleState::ROLE_STATE_CHANGING));
}
return ret;
}
int ObDupTableLSRoleStateHelper::restore_state(const ObDupTableLSRoleState &target_state,
ObDupTableLSRoleStateContainer &restore_state)
{
int ret = OB_SUCCESS;
DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)
if (OB_FAIL(ret)) {
// do nothing
} else if (ObDupTableLSRoleState::ROLE_STATE_CHANGING != cur_ls_state) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid cur ls state", K(ret), K(cur_ls_state), KPC(this));
} else {
ATOMIC_STORE(state_member, *backup_state_member);
}
return ret;
}
int ObDupTableLSRoleStateHelper::state_change_succ(const ObDupTableLSRoleState &target_state,
ObDupTableLSRoleStateContainer &restore_state)
{
int ret = OB_SUCCESS;
DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)
if (OB_FAIL(ret)) {
// do nothing
} else if (ObDupTableLSRoleState::ROLE_STATE_CHANGING != cur_ls_state
&& target_state != cur_ls_state) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid cur ls state", K(ret), K(cur_ls_state), KPC(this));
} else {
ATOMIC_STORE(state_member, static_cast<int64_t>(target_state));
}
return ret;
}
/*******************************************************
* HashMapTool (not thread safe)
*******************************************************/
@ -274,6 +432,29 @@ int ObDupTableLSCheckpoint::flush()
return ret;
}
int ObDupTableLSCheckpoint::offline()
{
int ret = OB_SUCCESS;
// SpinWLockGuard w_guard(ckpt_rw_lock_);
// DUP_TABLE_LOG(INFO, , args...)
return ret;
}
int ObDupTableLSCheckpoint::online()
{
int ret = OB_SUCCESS;
SpinWLockGuard w_guard(ckpt_rw_lock_);
lease_log_rec_scn_.reset();
start_replay_scn_.reset();
DUP_TABLE_LOG(INFO, "Dup Table LS Checkpoint Online", K(ret), KPC(this));
return ret;
}
/*******************************************************
* Dup_Table Log
*******************************************************/
@ -413,6 +594,11 @@ int ObDupTableLogOperator::deserialize_log_entry()
bool ObDupTableLogOperator::is_busy()
{
SpinRLockGuard guard(log_lock_);
return check_is_busy_without_lock();
}
bool ObDupTableLogOperator::check_is_busy_without_lock()
{
return !logging_tablet_set_ids_.empty() || !logging_lease_addrs_.empty();
}

89
src/storage/tx/ob_dup_table_base.h
View File

@ -87,6 +87,86 @@ struct DupTableInterfaceStat
K(dup_table_redo_sync_succ_cnt_),
K(dup_table_redo_sync_fail_cnt_));
};
/*******************************************************
* Dup_Table LS Role State
*******************************************************/
enum class ObDupTableLSRoleState : int64_t
{
UNKNOWN = 0,
ROLE_STATE_CHANGING = 1, // Follower
LS_REVOKE_SUCC = 10,
LS_TAKEOVER_SUCC = 11,
LS_OFFLINE_SUCC = 20,
LS_ONLINE_SUCC = 21,
LS_STOP_SUCC = 30,
LS_START_SUCC = 31,
};
struct ObDupTableLSRoleStateContainer
{
int64_t role_state_;
int64_t offline_state_;
int64_t stop_state_;
void reset()
{
role_state_ = static_cast<int64_t>(ObDupTableLSRoleState::UNKNOWN);
offline_state_ = static_cast<int64_t>(ObDupTableLSRoleState::UNKNOWN);
stop_state_ = static_cast<int64_t>(ObDupTableLSRoleState::UNKNOWN);
}
int64_t *get_target_state_ref(ObDupTableLSRoleState target_state);
bool check_target_state(ObDupTableLSRoleState target_state);
TO_STRING_KV(K(role_state_), K(offline_state_), K(stop_state_));
};
class ObDupTableLSRoleStateHelper
{
public:
ObDupTableLSRoleStateHelper(const char *module_name) : module_name_(module_name)
{
cur_state_.reset();
}
void reset()
{
cur_state_.role_state_ = static_cast<int64_t>(ObDupTableLSRoleState::LS_REVOKE_SUCC);
cur_state_.stop_state_ = static_cast<int64_t>(ObDupTableLSRoleState::LS_STOP_SUCC);
cur_state_.offline_state_ = static_cast<int64_t>(ObDupTableLSRoleState::LS_ONLINE_SUCC);
}
bool is_leader();
bool is_follower();
bool is_offline();
bool is_online();
bool is_stopped();
bool is_started();
bool is_leader_serving() { return is_leader() && is_online() && is_started(); }
bool is_follower_serving() { return is_follower() && is_online() && is_started(); }
bool is_active_ls() { return is_online() && is_started(); }
int prepare_state_change(const ObDupTableLSRoleState &target_state,
ObDupTableLSRoleStateContainer &restore_state);
int restore_state(const ObDupTableLSRoleState &target_state,
ObDupTableLSRoleStateContainer &restore_state);
int state_change_succ(const ObDupTableLSRoleState &target_state,
ObDupTableLSRoleStateContainer &restore_state);
TO_STRING_KV("module_name", module_name_, K(cur_state_));
private:
ObDupTableLSRoleStateContainer cur_state_;
const char *module_name_;
};
/*******************************************************
* HashMapTool (not thread safe)
*******************************************************/
@ -781,6 +861,9 @@ public:
int flush();
int offline();
int online();
void reset()
{
dup_ls_meta_.reset();
@ -863,6 +946,11 @@ public:
bool is_busy();
bool check_is_busy_without_lock();
void rlock_for_log() { log_lock_.rdlock(); }
void unlock_for_log() { log_lock_.unlock(); }
int on_success();
int on_failure();
@ -879,6 +967,7 @@ public:
K(logging_lease_addrs_),
K(logging_scn_),
K(logging_lsn_));
private:
static const int64_t MAX_LOG_BLOCK_SIZE = common::OB_MAX_LOG_ALLOWED_SIZE;
int prepare_serialize_log_entry_(int64_t &max_ser_size, DupLogTypeArray &type_array);

9
src/storage/tx/ob_dup_table_lease.cpp
View File

@ -46,6 +46,15 @@ int ObDupTableLSLeaseMgr::init(ObDupTableLSHandler *dup_ls_handle)
return ret;
}
int ObDupTableLSLeaseMgr::offline()
{
int ret = OB_SUCCESS;
follower_lease_info_.reset();
leader_lease_map_.clear();
return ret;
}
void ObDupTableLSLeaseMgr::reset()
{
ls_id_.reset();

1
src/storage/tx/ob_dup_table_lease.h
View File

@ -47,6 +47,7 @@ public:
ObDupTableLSLeaseMgr() : lease_diag_info_log_buf_(nullptr) { reset(); }
int init(ObDupTableLSHandler *dup_ls_handle);
int offline();
void destroy() { reset(); };
void reset();
bool is_master() { return ATOMIC_LOAD(&is_master_); }

17
src/storage/tx/ob_dup_table_tablets.cpp
View File

@ -414,6 +414,23 @@ void ObLSDupTabletsMgr::reset()
tablet_diag_info_log_buf_ = nullptr;
}
int ObLSDupTabletsMgr::offline()
{
int ret = OB_SUCCESS;
// DUP_TABLE_LOG(INFO, "before dup_tablets_mgr offline", K(ret),KPC(this));
if (OB_FAIL(clean_unlog_tablets_())) {
DUP_TABLE_LOG(WARN, "clean unlog tablets failed", K(ret), KPC(this));
} else if (OB_FAIL(clean_durable_confirming_tablets_(share::SCN::max_scn()))) {
DUP_TABLE_LOG(WARN, "clean durable confirming tablets failed", K(ret), KPC(this));
} else if (OB_FAIL(clean_readable_tablets_(share::SCN::max_scn()))) {
DUP_TABLE_LOG(WARN, "clean all readable tablets failed", K(ret), KPC(this));
}
return ret;
}
int ObLSDupTabletsMgr::check_readable(const common::ObTabletID &tablet_id,
bool &readable,
const share::SCN &snapshot,

4
src/storage/tx/ob_dup_table_tablets.h
View File

@ -1,4 +1,4 @@
// Copyrigh (c) 2021 OceanBase
// Copyrigh(c) 2021 OceanBase
// OceanBase is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
@ -502,6 +502,8 @@ public:
bool is_master() { return ATOMIC_LOAD(&is_master_); }
int offline();
const static int64_t MAX_CONFIRMING_TABLET_COUNT;
public:

11
src/storage/tx/ob_dup_table_ts_sync.cpp
View File

@ -45,6 +45,17 @@ int ObDupTableLSTsSyncMgr::init(ObDupTableLSHandler *dup_ls_handle)
return ret;
}
int ObDupTableLSTsSyncMgr::offline()
{
int ret = OB_SUCCESS;
if (OB_FAIL(clean_ts_info_cache_())) {
DUP_TABLE_LOG(WARN, "clean ts info cache", K(ret), KPC(this));
}
return ret;
}
int ObDupTableLSTsSyncMgr::validate_replay_ts(const common::ObAddr &dst,
const share::SCN &target_replay_scn,
const ObTransID &tx_id,

3
src/storage/tx/ob_dup_table_ts_sync.h
View File

@ -61,6 +61,7 @@ public:
public:
ObDupTableLSTsSyncMgr() : ts_sync_diag_info_log_buf_(nullptr) {}
int init(ObDupTableLSHandler *dup_ls_handle);
int offline();
void reset()
{
@ -109,6 +110,8 @@ public:
int get_lease_mgr_stat(ObDupLSLeaseMgrStatIterator &collect_iter,
FollowerLeaseMgrStatArr &arr);
TO_STRING_KV(K(ls_id_), K(ts_info_cache_.size()));
private:
int clean_ts_info_cache_();
int request_ts_info_by_rpc_(const common::ObAddr &addr, const share::SCN &leader_commit_scn);

493
src/storage/tx/ob_dup_table_util.cpp
View File

@ -286,8 +286,12 @@ int ObDupTabletScanTask::execute_for_dup_ls_()
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid ls ptr", K(ret), KP(cur_ls_ptr));
} else if (!cur_ls_ptr->get_dup_table_ls_handler()->is_master()) {
// do nothing
DUP_TABLE_LOG(DEBUG, "ls not leader", K(cur_ls_ptr->get_ls_id()));
// #ifndef NDEBUG
DUP_TABLE_LOG(INFO,
"ls not leader",
K(cur_ls_ptr->get_ls_id()),
KPC(cur_ls_ptr->get_dup_table_ls_handler()));
// #endif
} else if (OB_FAIL(refresh_dup_tablet_schema_(need_refreh_dup_schema, tenant_dup_tablet_set,
dup_ls_status_info))) {
DUP_TABLE_LOG(INFO, "refresh dup table schema failed", K(ret));
@ -371,7 +375,7 @@ int ObDupTableLSHandler::init(bool is_dup_table)
{
int ret = OB_SUCCESS;
if (is_dup_table) {
if (ATOMIC_LOAD(&is_inited_)) {
if (is_inited()) {
ret = OB_INIT_TWICE;
} else {
// init by dup_tablet_scan_task_.
@ -397,7 +401,7 @@ int ObDupTableLSHandler::init(bool is_dup_table)
DUP_TABLE_LOG(WARN, "init ts_sync_mgr failed", K(ret));
} else if (OB_FAIL(tablets_mgr_ptr_->init(this))) {
DUP_TABLE_LOG(WARN, "init tablets_mgr failed", K(ret));
} else if (ATOMIC_LOAD(&is_master_) && OB_FAIL(leader_takeover_(true /*is_resume*/))) {
} else if (ls_state_helper_.is_leader() && OB_FAIL(leader_takeover_(true /*is_resume*/))) {
DUP_TABLE_LOG(WARN, "leader takeover in init failed", K(ret));
} else {
ATOMIC_STORE(&is_inited_, true);
@ -421,20 +425,146 @@ int ObDupTableLSHandler::init(bool is_dup_table)
ts_sync_mgr_ptr_ = nullptr;
tablets_mgr_ptr_ = nullptr;
}
DUP_TABLE_LOG(INFO, "ls handler init", K(ret), K(ls_id_), K(is_master()), K(is_follower()),
KPC(lease_mgr_ptr_), KPC(tablets_mgr_ptr_), KP(ts_sync_mgr_ptr_));
DUP_TABLE_LOG(INFO, "ls handler init", K(ret), KPC(this));
}
}
return ret;
}
void ObDupTableLSHandler::start()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_START_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_START_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
}
}
void ObDupTableLSHandler::stop()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_STOP_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_STOP_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
}
}
int ObDupTableLSHandler::offline()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_OFFLINE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
} else if (OB_NOT_NULL(log_operator_) && log_operator_->is_busy()) {
ret = OB_EAGAIN;
DUP_TABLE_LOG(WARN, "wait log synced before offline", K(ret), KPC(this));
} else if (OB_NOT_NULL(tablets_mgr_ptr_) && OB_FAIL(tablets_mgr_ptr_->offline())) {
DUP_TABLE_LOG(WARN, "dup tablets mgr offline failed", K(ret), KPC(this));
} else if (OB_NOT_NULL(lease_mgr_ptr_) && OB_FAIL(lease_mgr_ptr_->offline())) {
DUP_TABLE_LOG(WARN, "dup lease mgr offline failed", K(ret), KPC(this));
} else if (OB_NOT_NULL(ts_sync_mgr_ptr_) && OB_FAIL(ts_sync_mgr_ptr_->offline())) {
DUP_TABLE_LOG(WARN, "dup ts mgr offline failed", K(ret), KPC(this));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_OFFLINE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
}
return ret;
}
int ObDupTableLSHandler::online()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_ONLINE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(dup_ls_ckpt_.online())) {
DUP_TABLE_LOG(WARN, "dup table checkpoint online failed", K(ret), KPC(this));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_ONLINE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
}
return ret;
}
int ObDupTableLSHandler::safe_to_destroy(bool &is_dup_table_handler_safe)
{
int ret = OB_SUCCESS;
is_dup_table_handler_safe = false;
if (OB_NOT_NULL(log_operator_) && log_operator_->is_busy()) {
ret = OB_EAGAIN;
DUP_TABLE_LOG(WARN, "wait log synced before destroy", K(ret), KPC(this));
}
if (OB_SUCC(ret)) {
is_dup_table_handler_safe = true;
}
return ret;
}
void ObDupTableLSHandler::destroy() { reset(); }
// int ObDupTableLSHandler::offline()
// {
// int ret = OB_SUCCESS;
// }
// return ret;
// }
//
// int ObDupTableLSHandler::online() {}
void ObDupTableLSHandler::reset()
{
// ATOMIC_STORE(&is_inited_, false);
is_inited_ = false;
is_master_ = false;
ls_state_helper_.reset();
dup_ls_ckpt_.reset();
@ -470,52 +600,76 @@ void ObDupTableLSHandler::reset()
}
}
bool ObDupTableLSHandler::is_master()
{
bool sub_master = true;
if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
sub_master = sub_master && ts_sync_mgr_ptr_->is_master();
}
if (OB_NOT_NULL(lease_mgr_ptr_)) {
sub_master = sub_master && lease_mgr_ptr_->is_master();
}
if (OB_NOT_NULL(tablets_mgr_ptr_)) {
sub_master = sub_master && tablets_mgr_ptr_->is_master();
}
// bool ObDupTableLSHandler::is_master()
// {
// bool sub_master = true;
// if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
// sub_master = sub_master && ts_sync_mgr_ptr_->is_master();
// }
// if (OB_NOT_NULL(lease_mgr_ptr_)) {
// sub_master = sub_master && lease_mgr_ptr_->is_master();
// }
// if (OB_NOT_NULL(tablets_mgr_ptr_)) {
// sub_master = sub_master && tablets_mgr_ptr_->is_master();
// }
//
// return (ATOMIC_LOAD(&is_master_)) && sub_master;
// }
return (ATOMIC_LOAD(&is_master_)) && sub_master;
// bool ObDupTableLSHandler::is_follower()
// {
// bool sub_not_master = true;
// if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
// sub_not_master = sub_not_master && !ts_sync_mgr_ptr_->is_master();
// }
// if (OB_NOT_NULL(lease_mgr_ptr_)) {
// sub_not_master = sub_not_master && !lease_mgr_ptr_->is_master();
// }
// if (OB_NOT_NULL(tablets_mgr_ptr_)) {
// sub_not_master = sub_not_master && !tablets_mgr_ptr_->is_master();
// }
//
// return (!ATOMIC_LOAD(&is_master_)) && sub_not_master;
// }
bool ObDupTableLSHandler::is_inited()
{
return ATOMIC_LOAD(&is_inited_);
}
bool ObDupTableLSHandler::is_follower()
{
bool sub_not_master = true;
if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
sub_not_master = sub_not_master && !ts_sync_mgr_ptr_->is_master();
}
if (OB_NOT_NULL(lease_mgr_ptr_)) {
sub_not_master = sub_not_master && !lease_mgr_ptr_->is_master();
}
if (OB_NOT_NULL(tablets_mgr_ptr_)) {
sub_not_master = sub_not_master && !tablets_mgr_ptr_->is_master();
}
bool ObDupTableLSHandler::is_master() { return ls_state_helper_.is_leader_serving(); }
return (!ATOMIC_LOAD(&is_master_)) && sub_not_master;
}
// bool ObDupTableLSHandler::is_online()
// {
// bool sub_online = true;
// if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
// sub_online = sub_online && !ts_sync_mgr_ptr_->is_master();
// }
// if (OB_NOT_NULL(lease_mgr_ptr_)) {
// sub_online = sub_online && !lease_mgr_ptr_->is_master();
// }
// if (OB_NOT_NULL(tablets_mgr_ptr_)) {
// sub_online = sub_online && !tablets_mgr_ptr_->is_master();
// }
//
// }
int ObDupTableLSHandler::ls_loop_handle()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
// TODO check stopped
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablets_mgr_ptr_)
if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablets_mgr_ptr_)
|| OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "dup table ls handle not init", K(ret));
} else if (!ls_state_helper_.is_active_ls()) {
ret = OB_LS_OFFLINE;
DUP_TABLE_LOG(WARN, "the ls is not active", K(ret), KPC(this));
} else if (!has_dup_tablet()) {
ret = OB_NO_TABLET;
DUP_TABLE_LOG(INFO, "no dup tablet, no need to do loop worker", K(ret), KPC(tablets_mgr_ptr_));
} else {
if (is_master()) {
if (ls_state_helper_.is_leader()) {
if (OB_ISNULL(log_operator_) || !log_operator_->is_busy()) {
// handle lease request and collect follower info
DupTableTsInfo min_lease_ts_info;
@ -539,16 +693,20 @@ int ObDupTableLSHandler::ls_loop_handle()
DUP_TABLE_LOG(WARN, "update all ts info cache failed", K(tmp_ret));
}
} else if (is_follower()) {
} else if (ls_state_helper_.is_follower()) {
if (OB_FAIL(lease_mgr_ptr_->follower_handle())) {
DUP_TABLE_LOG(WARN, "follower lease handle failed", K(ret));
}
} else {
DUP_TABLE_LOG(INFO, "undefined role state for dup table ls, skip this loop", K(ret),
K(ls_id_), K(ls_state_helper_));
}
DUP_TABLE_LOG(DEBUG, "loop running : dup table ls handler", K(ret), K(ls_id_), K(is_master()),
KPC(lease_mgr_ptr_), KPC(tablets_mgr_ptr_), KPC(log_operator_));
DUP_TABLE_LOG(DEBUG, "loop running : dup table ls handler", K(ret), K(ls_id_),
K(ls_state_helper_), KPC(lease_mgr_ptr_), KPC(tablets_mgr_ptr_),
KPC(log_operator_));
const int64_t fast_cur_time = ObTimeUtility::fast_current_time();
const bool is_leader = is_master();
const bool is_leader = ls_state_helper_.is_leader();
if (fast_cur_time - last_diag_info_print_us_[DupTableDiagStd::TypeIndex::LEASE_INDEX]
>= DupTableDiagStd::DUP_DIAG_PRINT_INTERVAL[DupTableDiagStd::TypeIndex::LEASE_INDEX]) {
@ -585,15 +743,20 @@ int ObDupTableLSHandler::refresh_dup_table_tablet(common::ObTabletID tablet_id,
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(tablets_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(tablets_mgr_ptr_)) {
ret = OB_NOT_INIT;
if (is_dup_table) {
DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_),
KP(tablets_mgr_ptr_));
DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_), K(is_dup_table),
KPC(this));
}
} else if (!ls_state_helper_.is_leader()) {
ret = OB_NOT_MASTER;
DUP_TABLE_LOG(WARN, "No need to refresh dup table schema", K(ret), K(tablet_id),
K(is_dup_table), KPC(this));
} else if (OB_FAIL(tablets_mgr_ptr_->refresh_dup_tablet(tablet_id, is_dup_table, refresh_time))) {
if (ret != OB_NOT_MASTER) {
DUP_TABLE_LOG(WARN, "refresh dup table tablet failed", K(ret), K(tablet_id), K(is_dup_table));
DUP_TABLE_LOG(WARN, "refresh dup table tablet failed", K(ret), K(tablet_id), K(is_dup_table),
KPC(this));
} else {
ret = OB_SUCCESS;
}
@ -605,9 +768,12 @@ int ObDupTableLSHandler::refresh_dup_table_tablet(common::ObTabletID tablet_id,
int ObDupTableLSHandler::recive_lease_request(const ObDupTableLeaseRequest &lease_req)
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(lease_mgr_ptr_));
} else if (!ls_state_helper_.is_leader_serving()) {
ret = OB_NOT_MASTER;
DUP_TABLE_LOG(WARN, "No need to recive lease request", K(ret), K(lease_req), KPC(this));
} else if (OB_FAIL(ts_sync_mgr_ptr_->handle_ts_sync_response(lease_req))) {
DUP_TABLE_LOG(WARN, "handle ts sync response failed", K(ret));
} else if (OB_FAIL(lease_mgr_ptr_->recive_lease_request(lease_req))) {
@ -620,9 +786,12 @@ int ObDupTableLSHandler::handle_ts_sync_response(const ObDupTableTsSyncResponse
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
} else if (!ls_state_helper_.is_leader_serving()) {
ret = OB_NOT_MASTER;
DUP_TABLE_LOG(WARN, "No need to handle ts sync response", K(ret), K(ts_sync_resp), KPC(this));
} else if (OB_FAIL(ts_sync_mgr_ptr_->handle_ts_sync_response(ts_sync_resp))) {
DUP_TABLE_LOG(WARN, "handle ts sync response failed", K(ret));
}
@ -634,9 +803,12 @@ int ObDupTableLSHandler::handle_ts_sync_request(const ObDupTableTsSyncRequest &t
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
} else if (!ls_state_helper_.is_follower_serving()) {
ret = OB_NOT_FOLLOWER;
DUP_TABLE_LOG(WARN, "No need to handle ts sync request", K(ret), K(ts_sync_req), KPC(this));
} else if (OB_FAIL(ts_sync_mgr_ptr_->handle_ts_sync_request(ts_sync_req))) {
DUP_TABLE_LOG(WARN, "handle ts sync request failed", K(ret));
}
@ -663,7 +835,7 @@ int ObDupTableLSHandler::check_redo_sync_completed(const ObTransID &tx_id,
before_prepare_gts.set_invalid();
int64_t start_us = OB_INVALID_TIMESTAMP;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_), KP(lease_mgr_ptr_),
KP(ts_sync_mgr_ptr_));
@ -775,7 +947,7 @@ int ObDupTableLSHandler::block_confirm_with_dup_tablet_change_snapshot(
ATOMIC_INC(&total_block_confirm_ref_);
if (!ATOMIC_LOAD(&is_inited_)) {
if (!is_inited()) {
// do nothing
ret = OB_SUCCESS;
} else {
@ -788,7 +960,7 @@ int ObDupTableLSHandler::gc_dup_tablets(const int64_t gc_ts, const int64_t max_t
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_)) {
if (!is_inited() || !ls_state_helper_.is_leader_serving()) {
// do nothing
} else if (OB_ISNULL(tablets_mgr_ptr_)) {
ret = OB_NOT_INIT;
@ -808,7 +980,7 @@ int ObDupTableLSHandler::try_to_confirm_tablets_(const share::SCN &confirm_scn)
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(tablets_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(tablets_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_),
KP(tablets_mgr_ptr_));
@ -871,6 +1043,9 @@ int ObDupTableLSHandler::check_dup_tablet_readable(const ObTabletID &tablet_id,
} else if (OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablets_mgr_ptr_)) {
// no dup tablet in ls
readable = false;
} else if (!ls_state_helper_.is_active_ls()) {
readable = false;
DUP_TABLE_LOG(INFO, "the ls is not active", K(ret), KPC(this));
} else if (!has_dup_tablet()) {
readable = false;
interface_stat_.dup_table_follower_read_tablet_not_exist_cnt_++;
@ -905,6 +1080,7 @@ int ObDupTableLSHandler::check_dup_tablet_readable(const ObTabletID &tablet_id,
return ret;
}
int64_t ObDupTableLSHandler::get_dup_tablet_count()
{
int64_t dup_tablet_cnt = 0;
@ -933,9 +1109,12 @@ int ObDupTableLSHandler::get_local_ts_info(DupTableTsInfo &ts_info)
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
} else if (!ls_state_helper_.is_active_ls()) {
ret = OB_LS_OFFLINE;
DUP_TABLE_LOG(WARN, "the ls is not active", K(ret), KPC(this));
} else if (OB_FAIL(ts_sync_mgr_ptr_->get_local_ts_info(ts_info))) {
DUP_TABLE_LOG(WARN, "get local ts sync info failed", K(ret));
}
@ -947,9 +1126,13 @@ int ObDupTableLSHandler::get_cache_ts_info(const common::ObAddr &addr, DupTableT
{
int ret = OB_SUCCESS;
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
} else if (!ls_state_helper_.is_leader_serving()) {
ret = OB_NOT_MASTER;
DUP_TABLE_LOG(WARN, "we can not get cached ts info from a follower", K(ret), K(ts_info),
KPC(this));
} else if (OB_FAIL(ts_sync_mgr_ptr_->get_cache_ts_info(addr, ts_info))) {
DUP_TABLE_LOG(WARN, "get cache ts info failed", K(ret), K(addr), K(ts_info));
}
@ -967,7 +1150,7 @@ int ObDupTableLSHandler::replay(const void *buffer,
const bool no_dup_tablet_before_replay = !has_dup_tablet();
// cover lease list and tablets list
if (!ATOMIC_LOAD(&is_inited_) && OB_FAIL(init(true))) {
if (!is_inited() && OB_FAIL(init(true))) {
DUP_TABLE_LOG(WARN, "init dup_ls_handle in replay failed", K(ret));
} else if (OB_FAIL(prepare_log_operator_())) {
DUP_TABLE_LOG(WARN, "init dup_table log operator failed", K(ret));
@ -1011,11 +1194,25 @@ int ObDupTableLSHandler::replay(const void *buffer,
void ObDupTableLSHandler::switch_to_follower_forcedly()
{
int ret = OB_SUCCESS;
if (OB_FAIL(leader_revoke_())) {
DUP_TABLE_LOG(ERROR, "switch to follower forcedly failed for dup table", K(ret), K(ls_id_),
K(is_master()));
int tmp_ret = OB_SUCCESS;
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_REVOKE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(leader_revoke_(true /*forcedly*/))) {
DUP_TABLE_LOG(ERROR, "switch to follower forcedly failed for dup table", K(ret), KPC(this));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_REVOKE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
}
ATOMIC_STORE(&is_master_, false);
}
int ObDupTableLSHandler::switch_to_follower_gracefully()
@ -1023,20 +1220,34 @@ int ObDupTableLSHandler::switch_to_follower_gracefully()
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (OB_FAIL(leader_revoke_())) {
DUP_TABLE_LOG(WARN, "switch to follower gracefully failed for dup table", K(ret), K(ls_id_),
K(is_master()));
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_REVOKE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(leader_revoke_(false /*forcedly*/))) {
DUP_TABLE_LOG(WARN, "switch to follower gracefully failed for dup table", K(ret), KPC(this));
}
ATOMIC_STORE(&is_master_, false);
if (OB_FAIL(ret)) {
if (OB_TMP_FAIL(resume_leader())) {
ret = OB_LS_NEED_REVOKE;
DUP_TABLE_LOG(WARN, "resume leader failed, need revoke", K(ret), K(tmp_ret), K(ls_id_));
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_REVOKE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
} else {
DUP_TABLE_LOG(WARN, "resume leader successfully, return error code", K(ret), K(tmp_ret),
K(ls_id_));
tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(leader_takeover_(true /*is_resume*/))) {
DUP_TABLE_LOG(WARN, "resume leader failed", K(ret), K(tmp_ret), KPC(this));
}
if (OB_SUCCESS != tmp_ret) {
ret = OB_LS_NEED_REVOKE;
DUP_TABLE_LOG(WARN, "resume leader failed, need revoke", K(ret), K(tmp_ret), KPC(this));
} else if (OB_TMP_FAIL(ls_state_helper_.restore_state(ObDupTableLSRoleState::LS_REVOKE_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "restore ls role state error", K(ret), KPC(this));
}
}
@ -1046,62 +1257,115 @@ int ObDupTableLSHandler::switch_to_follower_gracefully()
int ObDupTableLSHandler::resume_leader()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
const bool is_resume = true;
if (OB_FAIL(leader_takeover_(is_resume))) {
DUP_TABLE_LOG(WARN, "resume leader failed for dup table", K(ret), K(ls_id_), K(is_master()));
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(leader_takeover_(is_resume))) {
DUP_TABLE_LOG(WARN, "resume leader failed for dup table", K(ret), KPC(this));
}
ATOMIC_STORE(&is_master_, true);
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
} else {
tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(leader_revoke_(true /*forcedly*/))) {
DUP_TABLE_LOG(WARN, "leader revoke failed", K(ret), K(tmp_ret), KPC(this));
} else if (OB_TMP_FAIL(ls_state_helper_.restore_state(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "restore ls role state error", K(ret), KPC(this));
}
}
return ret;
}
int ObDupTableLSHandler::switch_to_leader()
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
const bool is_resume = false;
if (OB_FAIL(leader_takeover_(is_resume))) {
DUP_TABLE_LOG(WARN, "switch to leader failed for dup table", K(ret), K(ls_id_), K(is_master()));
ObDupTableLSRoleStateContainer restore_state_container;
if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
if (OB_NO_NEED_UPDATE == ret) {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(leader_takeover_(is_resume))) {
DUP_TABLE_LOG(WARN, "leader takeover failed for dup table", K(ret), KPC(this));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
}
} else {
tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(leader_revoke_(true /*forcedly*/))) {
DUP_TABLE_LOG(WARN, "leader revoke failed", K(ret), K(tmp_ret), KPC(this));
} else if (OB_TMP_FAIL(ls_state_helper_.restore_state(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
restore_state_container))) {
DUP_TABLE_LOG(ERROR, "restore ls role state error", K(ret), KPC(this));
}
}
ATOMIC_STORE(&is_master_, true);
return ret;
}
int ObDupTableLSHandler::leader_revoke_()
int ObDupTableLSHandler::leader_revoke_(const bool is_forcedly)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
if (!is_master()) {
// ret = OB_STATE_NOT_MATCH;
DUP_TABLE_LOG(WARN, "ObDupTableLSHandler has already been follower",
K(ret), K(ls_id_), K(is_master()));
} else {
// clean new/old tablet set
bool is_logging = false;
if (OB_NOT_NULL(log_operator_)) {
if (OB_NOT_NULL(tablets_mgr_ptr_)
&& OB_FAIL(tablets_mgr_ptr_->leader_revoke(log_operator_->is_busy()))) {
DUP_TABLE_LOG(WARN, "clean unreadable tablet set failed", K(ret));
log_operator_->rlock_for_log();
is_logging = log_operator_->check_is_busy_without_lock();
}
if (OB_NOT_NULL(tablets_mgr_ptr_) && OB_TMP_FAIL(tablets_mgr_ptr_->leader_revoke(is_logging))) {
DUP_TABLE_LOG(WARN, "tablets_mgr switch to follower failed", K(ret), K(tmp_ret), KPC(this));
if (!is_forcedly) {
ret = tmp_ret;
}
}
if (OB_NOT_NULL(log_operator_)) {
log_operator_->unlock_for_log();
}
if (OB_SUCC(ret) && OB_NOT_NULL(ts_sync_mgr_ptr_)) {
if (OB_FAIL(ts_sync_mgr_ptr_->leader_revoke())) {
DUP_TABLE_LOG(WARN, "ts_sync_mgr switch to follower failed", K(ret));
if (OB_TMP_FAIL(ts_sync_mgr_ptr_->leader_revoke())) {
DUP_TABLE_LOG(WARN, "ts_sync_mgr switch to follower failed", K(ret), K(tmp_ret), KPC(this));
if (!is_forcedly) {
ret = tmp_ret;
}
}
}
if (OB_SUCC(ret) && OB_NOT_NULL(lease_mgr_ptr_)) {
if (OB_FAIL(lease_mgr_ptr_->leader_revoke())) {
DUP_TABLE_LOG(WARN, "lease_mgr switch to follower failed", K(ret));
if (OB_TMP_FAIL(lease_mgr_ptr_->leader_revoke())) {
DUP_TABLE_LOG(WARN, "lease_mgr switch to follower failed", K(ret), K(tmp_ret), KPC(this));
if (!is_forcedly) {
ret = tmp_ret;
}
}
}
interface_stat_.reset();
DUP_TABLE_LOG(INFO, "Leader Revoke", K(ret), K(ls_id_), KPC(log_operator_));
DUP_TABLE_LOG(INFO, "Leader Revoke", K(ret), K(is_forcedly), KPC(this));
return ret;
}
@ -1109,10 +1373,6 @@ int ObDupTableLSHandler::leader_takeover_(const bool is_resume)
{
int ret = OB_SUCCESS;
if (is_master()) {
ret = OB_STATE_NOT_MATCH;
DUP_TABLE_LOG(ERROR, "unexpected ObDupTableLSHandler role", K(ret), K(ls_id_), K(is_master()));
} else {
// clean ts info cache
if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
ts_sync_mgr_ptr_->leader_takeover();
@ -1128,10 +1388,9 @@ int ObDupTableLSHandler::leader_takeover_(const bool is_resume)
DUP_TABLE_LOG(WARN, "clean unreadable tablet set failed", K(ret));
}
}
}
interface_stat_.reset();
DUP_TABLE_LOG(INFO, "Leader Takeover", K(ret), K(ls_id_), K(is_resume));
DUP_TABLE_LOG(INFO, "Leader Takeover", K(ret), K(is_resume), KPC(this));
return ret;
}
@ -1154,6 +1413,40 @@ int ObDupTableLSHandler::prepare_log_operator_()
return ret;
}
int ObDupTableLSHandler::set_dup_table_ls_meta(
const ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta,
bool need_flush_slog)
{
int ret = OB_SUCCESS;
DUP_TABLE_LOG(INFO, "try to recover dup table ls meta", K(ret), K(need_flush_slog),
K(dup_ls_meta), KPC(this));
if (OB_FAIL(dup_ls_ckpt_.set_dup_ls_meta(dup_ls_meta))) {
DUP_TABLE_LOG(WARN, "set dup ls meta failed", K(ret), K(need_flush_slog), K(dup_ls_meta),
KPC(this));
} else if (need_flush_slog && OB_FAIL(dup_ls_ckpt_.flush())) {
DUP_TABLE_LOG(WARN, "flush slog failed", K(ret), K(need_flush_slog), K(dup_ls_meta), KPC(this));
}
return ret;
}
int ObDupTableLSHandler::flush(share::SCN &rec)
{
int ret = OB_SUCCESS;
if (!ls_state_helper_.is_started()) {
ret = OB_LS_NOT_EXIST;
DUP_TABLE_LOG(WARN, "this ls is not started", K(ret), K(rec), KPC(this));
} else {
ret = dup_ls_ckpt_.flush();
}
return ret;
}
int ObDupTableLSHandler::check_and_update_max_replayed_scn(const share::SCN &max_replayed_scn)
{
int ret = OB_SUCCESS;
@ -1186,7 +1479,7 @@ int ObDupTableLSHandler::get_min_lease_ts_info_(DupTableTsInfo &min_ts_info)
LeaseAddrArray lease_valid_array;
min_ts_info.reset();
if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid arguments", K(is_inited_), KP(lease_mgr_ptr_),
KP(ts_sync_mgr_ptr_));
@ -1226,7 +1519,7 @@ int ObDupTableLSHandler::get_lease_mgr_stat(ObDupLSLeaseMgrStatIterator &collect
FollowerLeaseMgrStatArr collect_arr;
// collect all leader info
if (is_master()) {
if (ls_state_helper_.is_leader_serving()) {
if (OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
ret = OB_NOT_INIT;
DUP_TABLE_LOG(WARN, "not init", K(ret), KPC(lease_mgr_ptr_), KP(ts_sync_mgr_ptr_));

44
src/storage/tx/ob_dup_table_util.h
View File

@ -14,6 +14,7 @@
// #include "lib/hash/ob_hashset.h"
#include "logservice/ob_log_base_type.h"
#include "storage/tx/ob_dup_table_tablets.h"
#include "storage/tx/ob_dup_table_lease.h"
#include "storage/tx/ob_trans_define.h"
#include "storage/tx/ob_dup_table_stat.h"
@ -96,17 +97,20 @@ class ObDupTableLSHandler : public logservice::ObIReplaySubHandler,
{
public:
ObDupTableLSHandler()
: lease_mgr_ptr_(nullptr), ts_sync_mgr_ptr_(nullptr), tablets_mgr_ptr_(nullptr),
log_operator_(nullptr)
: ls_state_helper_("DupTableLSHandler"), lease_mgr_ptr_(nullptr), ts_sync_mgr_ptr_(nullptr),
tablets_mgr_ptr_(nullptr), log_operator_(nullptr)
{
reset();
}
// init by ObDupTabletScanTask or replay
int init(bool is_dup_table);
void start();
void stop(); // TODO stop submit log before the log handler is invalid.
void wait(); // TODO
int safe_to_destroy(bool &is_dup_table_handler_safe); // TODO
void destroy(); // remove from dup_table_loop_worker
int offline();
int online();
void reset();
const share::ObLSID &get_ls_id() { return ls_id_; }
@ -119,7 +123,18 @@ public:
}
bool is_master();
bool is_follower();
// bool is_follower();
bool is_inited();
// bool is_online();
TO_STRING_KV(K(ls_id_),
K(ls_state_helper_),
K(dup_ls_ckpt_),
KPC(lease_mgr_ptr_),
KPC(tablets_mgr_ptr_),
// KPC(ts_sync_mgr_ptr_),
KPC(log_operator_),
K(interface_stat_));
public:
int ls_loop_handle();
@ -163,7 +178,6 @@ public:
bool &readable);
public:
bool is_inited() { return is_inited_; }
int64_t get_dup_tablet_count();
bool has_dup_tablet();
int gc_dup_tablets(const int64_t gc_ts, const int64_t max_task_interval);
@ -185,17 +199,15 @@ public:
int resume_leader();
int switch_to_leader();
int set_dup_table_ls_meta(const ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta)
{
return dup_ls_ckpt_.set_dup_ls_meta(dup_ls_meta);
}
int set_dup_table_ls_meta(const ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta,
bool need_flush_slog = false);
int get_dup_table_ls_meta(ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta) const
{
return dup_ls_ckpt_.get_dup_ls_meta(dup_ls_meta);
}
share::SCN get_rec_scn() { return dup_ls_ckpt_.get_lease_log_rec_scn(); }
int flush(share::SCN &rec) { return dup_ls_ckpt_.flush(); }
int flush(share::SCN &rec);
logservice::ObLogHandler *get_log_handler() { return log_handler_; }
@ -207,22 +219,24 @@ public:
int64_t get_total_block_confirm_ref() { return ATOMIC_LOAD(&total_block_confirm_ref_); }
int check_and_update_max_replayed_scn(const share::SCN &max_replayed_scn);
private:
DISALLOW_COPY_AND_ASSIGN(ObDupTableLSHandler);
int prepare_log_operator_();
int get_min_lease_ts_info_(DupTableTsInfo &min_ts_info);
int leader_takeover_(const bool is_resume);
int leader_revoke_();
int leader_revoke_(const bool is_forcedly);
int try_to_confirm_tablets_(const share::SCN &confirm_ts);
private:
share::ObLSID ls_id_;
bool is_master_; // set by role change
bool is_inited_; // ste by replay or dup_tablet_scan_task_
bool is_stopped_; // TODO
// TODO: is_stopped_;
bool is_inited_;
// set these flag for a normal ls without dup_table
ObDupTableLSRoleStateHelper ls_state_helper_;
int64_t total_block_confirm_ref_; // block new dup tablet confirmed