clear all memory state in offline for dup_table

src/storage/high_availability/ob_ls_migration.cpp

@@ -1253,7 +1253,8 @@ int ObStartMigrationTask::update_ls_()
     if (OB_FAIL(ls->update_ls_meta(update_restore_status,
                                    ctx_->src_ls_meta_package_.ls_meta_))) {
       LOG_WARN("failed to update ls meta", K(ret), KPC(ctx_));
-    } else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_))) {
+    } else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_,
+                                                 true /*need_flush_slog*/))) {
       LOG_WARN("failed to set dup table ls meta", K(ret), KPC(ctx_));
     } else if (OB_FAIL(ls->get_end_lsn(end_lsn))) {
       LOG_WARN("failed to get end lsn", K(ret), KPC(ctx_));

src/storage/high_availability/ob_ls_restore.cpp

@@ -1083,7 +1083,8 @@ int ObStartLSRestoreTask::update_ls_meta_()
   } else if (OB_FAIL(ls->update_ls_meta(false/*don't update restore status*/,
                                         ctx_->src_ls_meta_package_.ls_meta_))) {
     LOG_WARN("fail to update ls meta", K(ret), KPC(ls), KPC(ctx_));
-  } else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_))) {
+  } else if (OB_FAIL(ls->set_dup_table_ls_meta(ctx_->src_ls_meta_package_.dup_ls_meta_,
+                                               true /*need_flush_slog*/))) {
     LOG_WARN("fail to set dup table ls meta", K(ret), KPC(ctx_));
   } else {
     LOG_INFO("update ls meta succeed", KPC(ls), KPC(ctx_));

src/storage/ls/ob_ls.cpp

@@ -465,6 +465,7 @@ int ObLS::start()
     LOG_WARN("set max replay commit scn fail", K(ret), K(ls_meta_.get_clog_checkpoint_scn()));
   } else {
     checkpoint_executor_.start();
+    dup_table_ls_handler_.start();
     LOG_INFO("start_ls finish", KR(ret), KPC(this));
     // do nothing
   }
@@ -494,6 +495,7 @@ int ObLS::stop_()
   tx_table_.stop();
   ls_restore_handler_.stop();
   keep_alive_ls_handler_.stop();
+  dup_table_ls_handler_.stop();
   log_handler_.reset_election_priority();
   restore_handler_.stop();
   if (OB_FAIL(log_handler_.stop())) {
@@ -579,6 +581,7 @@ bool ObLS::safe_to_destroy()
   bool is_ls_restore_handler_safe = false;
   bool is_tablet_service_safe = false;
   bool is_data_check_point_safe = false;
+  bool is_dup_table_handler_safe = false;
   bool is_log_handler_safe = false;
 
   if (OB_FAIL(ls_tablet_svr_.safe_to_destroy(is_tablet_service_safe))) {
@@ -587,6 +590,9 @@ bool ObLS::safe_to_destroy()
   } else if (OB_FAIL(data_checkpoint_.safe_to_destroy(is_data_check_point_safe))) {
     LOG_WARN("data_checkpoint check safe to destroy failed", K(ret), KPC(this));
   } else if (!is_data_check_point_safe) {
+  } else if (OB_FAIL(dup_table_ls_handler_.safe_to_destroy(is_dup_table_handler_safe))) {
+    LOG_WARN("dup table ls handler safe to destroy failed", K(ret), KPC(this));
+  } else if (!is_dup_table_handler_safe) {
   } else if (OB_FAIL(ls_restore_handler_.safe_to_destroy(is_ls_restore_handler_safe))) {
     LOG_WARN("ls restore handler safe to destroy failed", K(ret), KPC(this));
   } else if (!is_ls_restore_handler_safe) {
@@ -605,6 +611,7 @@ bool ObLS::safe_to_destroy()
       if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) {
         LOG_WARN("this ls is not safe to destroy", K(is_safe),
                  K(is_tablet_service_safe), K(is_data_check_point_safe),
+                 K(is_dup_table_handler_safe),
                  K(is_ls_restore_handler_safe), K(is_log_handler_safe),
                  "ls_ref", ref_mgr_.get_total_ref_cnt(),
                  K(ret), KP(this), KPC(this));
@@ -782,6 +789,8 @@ int ObLS::offline_()
     LOG_WARN("ddl log handler offline failed", K(ret), K(ls_meta_));
   } else if (OB_FAIL(offline_tx_())) {
     LOG_WARN("offline tx service failed", K(ret), K(ls_meta_));
+  } else if (OB_FAIL(dup_table_ls_handler_.offline())) {
+    LOG_WARN("offline dup table ls handler failed", K(ret), K(ls_meta_));
   } else if (OB_FAIL(lock_table_.offline())) {
     LOG_WARN("lock table offline failed", K(ret), K(ls_meta_));
   // force release memtables created by force_tablet_freeze called during major
@@ -881,6 +890,8 @@ int ObLS::online()
     LOG_WARN("tablet service online failed", K(ret), K(ls_meta_));
   } else if (OB_FAIL(lock_table_.online())) {
     LOG_WARN("lock table online failed", K(ret), K(ls_meta_));
+  } else if (OB_FAIL(dup_table_ls_handler_.online())) {
+    LOG_WARN("dup table ls handler online failed", K(ret), K(ls_meta_));
   } else if (OB_FAIL(online_tx_())) {
     LOG_WARN("ls tx online failed", K(ret), K(ls_meta_));
   } else if (OB_FAIL(ls_ddl_log_handler_.online())) {

src/storage/tx/ob_dup_table_base.cpp

@@ -38,6 +38,164 @@ const int64_t DupTableDiagStd::DUP_DIAG_PRINT_INTERVAL[DupTableDiagStd::TypeInde
     3 * 60 * 1000 * 1000 // 3min , ts_sync_print_interval
 };
 
+/*******************************************************
+ *  Dup_Table LS Role State
+ *******************************************************/
+int64_t *ObDupTableLSRoleStateContainer::get_target_state_ref(ObDupTableLSRoleState target_state)
+{
+  int64_t *state_member = nullptr;
+  switch (target_state) {
+  case ObDupTableLSRoleState::LS_REVOKE_SUCC:
+  case ObDupTableLSRoleState::LS_TAKEOVER_SUCC: {
+    state_member = &role_state_;
+    break;
+  }
+  case ObDupTableLSRoleState::LS_OFFLINE_SUCC:
+  case ObDupTableLSRoleState::LS_ONLINE_SUCC: {
+    state_member = &offline_state_;
+    break;
+  }
+  case ObDupTableLSRoleState::LS_START_SUCC:
+  case ObDupTableLSRoleState::LS_STOP_SUCC: {
+    state_member = &stop_state_;
+    break;
+  }
+  default: {
+    state_member = nullptr;
+    DUP_TABLE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "undefined target role state", K(target_state),
+                      KPC(this));
+    break;
+  }
+  }
+  return state_member;
+}
+
+bool ObDupTableLSRoleStateContainer::check_target_state(ObDupTableLSRoleState target_state)
+{
+  bool is_same = false;
+  int64_t *state_member = get_target_state_ref(target_state);
+  if (OB_NOT_NULL(state_member)) {
+    ObDupTableLSRoleState cur_state = static_cast<ObDupTableLSRoleState>(ATOMIC_LOAD(state_member));
+    if (target_state == cur_state) {
+      is_same = true;
+    } else if (ObDupTableLSRoleState::ROLE_STATE_CHANGING == cur_state) {
+      is_same = false;
+      DUP_TABLE_LOG(INFO, "The Role State is Changing", K(is_same), K(cur_state), KPC(this));
+    } else {
+      is_same = false;
+    }
+  }
+  return is_same;
+}
+
+bool ObDupTableLSRoleStateHelper::is_leader()
+{
+  return  cur_state_.check_target_state(ObDupTableLSRoleState::LS_TAKEOVER_SUCC);
+}
+
+bool ObDupTableLSRoleStateHelper::is_follower()
+{
+  return  cur_state_.check_target_state(ObDupTableLSRoleState::LS_REVOKE_SUCC);
+}
+
+bool ObDupTableLSRoleStateHelper::is_offline()
+{
+  return cur_state_.check_target_state(ObDupTableLSRoleState::LS_OFFLINE_SUCC);
+}
+
+bool ObDupTableLSRoleStateHelper::is_online()
+{
+  return cur_state_.check_target_state(ObDupTableLSRoleState::LS_ONLINE_SUCC);
+}
+
+bool ObDupTableLSRoleStateHelper::is_stopped()
+{
+  return cur_state_.check_target_state(ObDupTableLSRoleState::LS_STOP_SUCC);
+}
+
+bool ObDupTableLSRoleStateHelper::is_started()
+{
+  return cur_state_.check_target_state(ObDupTableLSRoleState::LS_START_SUCC);
+}
+
+#define DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)                             \
+  int64_t *state_member = cur_state_.get_target_state_ref(target_state);                          \
+  int64_t *backup_state_member = restore_state.get_target_state_ref(target_state);                \
+  ObDupTableLSRoleState cur_ls_state = ObDupTableLSRoleState::UNKNOWN;                            \
+  if (OB_ISNULL(state_member) || OB_ISNULL(backup_state_member)) {                                \
+    ret = OB_INVALID_ARGUMENT;                                                                    \
+    DUP_TABLE_LOG(WARN, "invalid arguments", K(ret), KP(state_member), KP(backup_state_member), \
+                  K(target_state), KPC(this));                                             \
+  } else if (OB_FALSE_IT(cur_ls_state =                                                           \
+                             static_cast<ObDupTableLSRoleState>(ATOMIC_LOAD(state_member)))) {    \
+  }
+
+int ObDupTableLSRoleStateHelper::prepare_state_change(const ObDupTableLSRoleState &target_state,
+                                                      ObDupTableLSRoleStateContainer &restore_state)
+{
+  int ret = OB_SUCCESS;
+
+  DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)
+
+  if (OB_FAIL(ret)) {
+    // do nothing
+  } else if (target_state == cur_ls_state) {
+    ret = OB_NO_NEED_UPDATE;
+    DUP_TABLE_LOG(INFO, "the cur state has already been same as target_state", K(ret),
+                  K(target_state), KPC(this));
+  } else if (cur_ls_state == ObDupTableLSRoleState::UNKNOWN) {
+    ret = OB_INVALID_ARGUMENT;
+    DUP_TABLE_LOG(INFO, "invalid role state", K(cur_ls_state), KPC(this));
+  } else {
+    if (ObDupTableLSRoleState::ROLE_STATE_CHANGING == cur_ls_state) {
+      DUP_TABLE_LOG(INFO,
+                    "cur_ls_state is the ROLE_STATE_CHANGING, may be error in the last role change",
+                    K(ret), K(cur_ls_state), K(target_state), KPC(this));
+    }
+    *backup_state_member = static_cast<int64_t>(cur_ls_state);
+    ATOMIC_STORE(state_member, static_cast<int64_t>(ObDupTableLSRoleState::ROLE_STATE_CHANGING));
+  }
+
+  return ret;
+}
+
+int ObDupTableLSRoleStateHelper::restore_state(const ObDupTableLSRoleState &target_state,
+                                               ObDupTableLSRoleStateContainer &restore_state)
+{
+  int ret = OB_SUCCESS;
+  DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)
+
+  if (OB_FAIL(ret)) {
+    // do nothing
+  } else if (ObDupTableLSRoleState::ROLE_STATE_CHANGING != cur_ls_state) {
+    ret = OB_INVALID_ARGUMENT;
+    DUP_TABLE_LOG(WARN, "invalid cur ls state", K(ret), K(cur_ls_state), KPC(this));
+  } else {
+    ATOMIC_STORE(state_member, *backup_state_member);
+  }
+
+  return ret;
+}
+
+int ObDupTableLSRoleStateHelper::state_change_succ(const ObDupTableLSRoleState &target_state,
+                                                   ObDupTableLSRoleStateContainer &restore_state)
+{
+  int ret = OB_SUCCESS;
+  DUP_TABLE_LS_STATE_GET_AND_CHECK(target_state, restore_state)
+
+  if (OB_FAIL(ret)) {
+    // do nothing
+  } else if (ObDupTableLSRoleState::ROLE_STATE_CHANGING != cur_ls_state
+             && target_state != cur_ls_state) {
+    ret = OB_INVALID_ARGUMENT;
+    DUP_TABLE_LOG(WARN, "invalid cur ls state", K(ret), K(cur_ls_state), KPC(this));
+  } else {
+    ATOMIC_STORE(state_member, static_cast<int64_t>(target_state));
+  }
+
+  return ret;
+}
+
 /*******************************************************
  *  HashMapTool (not thread safe)
  *******************************************************/
@@ -274,6 +432,29 @@ int ObDupTableLSCheckpoint::flush()
   return ret;
 }
 
+int ObDupTableLSCheckpoint::offline()
+{
+  int ret = OB_SUCCESS;
+
+  // SpinWLockGuard w_guard(ckpt_rw_lock_);
+
+  // DUP_TABLE_LOG(INFO, , args...)
+
+  return ret;
+}
+
+int ObDupTableLSCheckpoint::online()
+{
+  int ret = OB_SUCCESS;
+  SpinWLockGuard w_guard(ckpt_rw_lock_);
+
+  lease_log_rec_scn_.reset();
+  start_replay_scn_.reset();
+
+  DUP_TABLE_LOG(INFO, "Dup Table LS Checkpoint Online", K(ret), KPC(this));
+  return ret;
+}
+
 /*******************************************************
  *  Dup_Table Log
  *******************************************************/
@@ -413,6 +594,11 @@ int ObDupTableLogOperator::deserialize_log_entry()
 bool ObDupTableLogOperator::is_busy()
 {
   SpinRLockGuard guard(log_lock_);
+  return check_is_busy_without_lock();
+}
+
+bool ObDupTableLogOperator::check_is_busy_without_lock()
+{
   return !logging_tablet_set_ids_.empty() || !logging_lease_addrs_.empty();
 }
 

src/storage/tx/ob_dup_table_base.h

@@ -87,6 +87,86 @@ struct DupTableInterfaceStat
                K(dup_table_redo_sync_succ_cnt_),
                K(dup_table_redo_sync_fail_cnt_));
 };
+
+/*******************************************************
+ *  Dup_Table LS Role State
+ *******************************************************/
+enum class ObDupTableLSRoleState : int64_t
+{
+  UNKNOWN = 0,
+
+  ROLE_STATE_CHANGING = 1, // Follower
+
+  LS_REVOKE_SUCC = 10,
+  LS_TAKEOVER_SUCC = 11,
+
+  LS_OFFLINE_SUCC = 20,
+  LS_ONLINE_SUCC = 21,
+
+  LS_STOP_SUCC = 30,
+  LS_START_SUCC = 31,
+};
+
+struct ObDupTableLSRoleStateContainer
+{
+  int64_t role_state_;
+  int64_t offline_state_;
+  int64_t stop_state_;
+
+  void reset()
+  {
+    role_state_ = static_cast<int64_t>(ObDupTableLSRoleState::UNKNOWN);
+    offline_state_ = static_cast<int64_t>(ObDupTableLSRoleState::UNKNOWN);
+    stop_state_ = static_cast<int64_t>(ObDupTableLSRoleState::UNKNOWN);
+  }
+
+  int64_t *get_target_state_ref(ObDupTableLSRoleState target_state);
+  bool check_target_state(ObDupTableLSRoleState target_state);
+
+  TO_STRING_KV(K(role_state_), K(offline_state_), K(stop_state_));
+};
+
+class ObDupTableLSRoleStateHelper
+{
+public:
+  ObDupTableLSRoleStateHelper(const char *module_name) : module_name_(module_name)
+  {
+    cur_state_.reset();
+  }
+
+  void reset()
+  {
+    cur_state_.role_state_ = static_cast<int64_t>(ObDupTableLSRoleState::LS_REVOKE_SUCC);
+    cur_state_.stop_state_ = static_cast<int64_t>(ObDupTableLSRoleState::LS_STOP_SUCC);
+    cur_state_.offline_state_ = static_cast<int64_t>(ObDupTableLSRoleState::LS_ONLINE_SUCC);
+  }
+
+  bool is_leader();
+  bool is_follower();
+  bool is_offline();
+  bool is_online();
+  bool is_stopped();
+  bool is_started();
+
+  bool is_leader_serving() { return is_leader() && is_online() && is_started(); }
+  bool is_follower_serving() { return is_follower() && is_online() && is_started(); }
+  bool is_active_ls() { return is_online() && is_started(); }
+
+  int prepare_state_change(const ObDupTableLSRoleState &target_state,
+                           ObDupTableLSRoleStateContainer &restore_state);
+  int restore_state(const ObDupTableLSRoleState &target_state,
+                           ObDupTableLSRoleStateContainer &restore_state);
+  int state_change_succ(const ObDupTableLSRoleState &target_state,
+                           ObDupTableLSRoleStateContainer &restore_state);
+
+  TO_STRING_KV("module_name", module_name_, K(cur_state_));
+
+private:
+  ObDupTableLSRoleStateContainer cur_state_;
+  const char *module_name_;
+};
+
+
 /*******************************************************
  *  HashMapTool (not thread safe)
  *******************************************************/
@@ -781,6 +861,9 @@ public:
 
   int flush();
 
+  int offline();
+  int online();
+
   void reset()
   {
     dup_ls_meta_.reset();
@@ -863,6 +946,11 @@ public:
 
   bool is_busy();
 
+  bool check_is_busy_without_lock();
+
+  void rlock_for_log() { log_lock_.rdlock(); }
+  void unlock_for_log() { log_lock_.unlock(); }
+
   int on_success();
   int on_failure();
 
@@ -879,6 +967,7 @@ public:
                K(logging_lease_addrs_),
                K(logging_scn_),
                K(logging_lsn_));
+
 private:
   static const int64_t MAX_LOG_BLOCK_SIZE = common::OB_MAX_LOG_ALLOWED_SIZE;
   int prepare_serialize_log_entry_(int64_t &max_ser_size, DupLogTypeArray &type_array);

src/storage/tx/ob_dup_table_lease.cpp

@@ -46,6 +46,15 @@ int ObDupTableLSLeaseMgr::init(ObDupTableLSHandler *dup_ls_handle)
   return ret;
 }
 
+int ObDupTableLSLeaseMgr::offline()
+{
+  int ret = OB_SUCCESS;
+
+  follower_lease_info_.reset();
+  leader_lease_map_.clear();
+  return ret;
+}
+
 void ObDupTableLSLeaseMgr::reset()
 {
   ls_id_.reset();

src/storage/tx/ob_dup_table_lease.h

@@ -47,6 +47,7 @@ public:
   ObDupTableLSLeaseMgr() : lease_diag_info_log_buf_(nullptr) { reset(); }
 
   int init(ObDupTableLSHandler *dup_ls_handle);
+  int offline();
   void destroy() { reset(); };
   void reset();
   bool is_master() { return ATOMIC_LOAD(&is_master_); }

src/storage/tx/ob_dup_table_tablets.cpp

@@ -414,6 +414,23 @@ void ObLSDupTabletsMgr::reset()
   tablet_diag_info_log_buf_ = nullptr;
 }
 
+int ObLSDupTabletsMgr::offline()
+{
+  int ret = OB_SUCCESS;
+
+  // DUP_TABLE_LOG(INFO, "before dup_tablets_mgr offline", K(ret),KPC(this));
+
+  if (OB_FAIL(clean_unlog_tablets_())) {
+    DUP_TABLE_LOG(WARN, "clean unlog tablets failed", K(ret), KPC(this));
+  } else if (OB_FAIL(clean_durable_confirming_tablets_(share::SCN::max_scn()))) {
+    DUP_TABLE_LOG(WARN, "clean durable confirming tablets failed", K(ret), KPC(this));
+  } else if (OB_FAIL(clean_readable_tablets_(share::SCN::max_scn()))) {
+    DUP_TABLE_LOG(WARN, "clean all readable tablets failed", K(ret), KPC(this));
+  }
+
+  return ret;
+}
+
 int ObLSDupTabletsMgr::check_readable(const common::ObTabletID &tablet_id,
                                       bool &readable,
                                       const share::SCN &snapshot,

src/storage/tx/ob_dup_table_tablets.h

@@ -1,4 +1,4 @@
-// Copyrigh (c) 2021 OceanBase
+// Copyrigh(c) 2021 OceanBase
 // OceanBase is licensed under Mulan PubL v2.
 // You can use this software according to the terms and conditions of the Mulan PubL v2.
 // You may obtain a copy of Mulan PubL v2 at:
@@ -502,6 +502,8 @@ public:
 
   bool is_master() { return ATOMIC_LOAD(&is_master_); }
 
+  int offline();
+
   const static int64_t MAX_CONFIRMING_TABLET_COUNT;
 
 public:

src/storage/tx/ob_dup_table_ts_sync.cpp

@@ -45,6 +45,17 @@ int ObDupTableLSTsSyncMgr::init(ObDupTableLSHandler *dup_ls_handle)
   return ret;
 }
 
+int ObDupTableLSTsSyncMgr::offline()
+{
+  int ret = OB_SUCCESS;
+
+  if (OB_FAIL(clean_ts_info_cache_())) {
+    DUP_TABLE_LOG(WARN, "clean ts info cache", K(ret), KPC(this));
+  }
+
+  return ret;
+}
+
 int ObDupTableLSTsSyncMgr::validate_replay_ts(const common::ObAddr &dst,
                                               const share::SCN &target_replay_scn,
                                               const ObTransID &tx_id,

src/storage/tx/ob_dup_table_ts_sync.h

@@ -61,6 +61,7 @@ public:
 public:
   ObDupTableLSTsSyncMgr() : ts_sync_diag_info_log_buf_(nullptr) {}
   int init(ObDupTableLSHandler *dup_ls_handle);
+  int offline();
 
   void reset()
   {
@@ -109,6 +110,8 @@ public:
 
   int get_lease_mgr_stat(ObDupLSLeaseMgrStatIterator &collect_iter,
                          FollowerLeaseMgrStatArr &arr);
+
+  TO_STRING_KV(K(ls_id_), K(ts_info_cache_.size()));
 private:
   int clean_ts_info_cache_();
   int request_ts_info_by_rpc_(const common::ObAddr &addr, const share::SCN &leader_commit_scn);

src/storage/tx/ob_dup_table_util.cpp

@@ -286,8 +286,12 @@ int ObDupTabletScanTask::execute_for_dup_ls_()
       ret = OB_INVALID_ARGUMENT;
       DUP_TABLE_LOG(WARN, "invalid ls ptr", K(ret), KP(cur_ls_ptr));
     } else if (!cur_ls_ptr->get_dup_table_ls_handler()->is_master()) {
-      // do nothing
+      // #ifndef NDEBUG
-      DUP_TABLE_LOG(DEBUG, "ls not leader", K(cur_ls_ptr->get_ls_id()));
+      DUP_TABLE_LOG(INFO,
+                    "ls not leader",
+                    K(cur_ls_ptr->get_ls_id()),
+                    KPC(cur_ls_ptr->get_dup_table_ls_handler()));
+      // #endif
     } else if (OB_FAIL(refresh_dup_tablet_schema_(need_refreh_dup_schema, tenant_dup_tablet_set,
                                                   dup_ls_status_info))) {
       DUP_TABLE_LOG(INFO, "refresh dup table schema failed", K(ret));
@@ -371,7 +375,7 @@ int ObDupTableLSHandler::init(bool is_dup_table)
 {
   int ret = OB_SUCCESS;
   if (is_dup_table) {
-    if (ATOMIC_LOAD(&is_inited_)) {
+    if (is_inited()) {
       ret = OB_INIT_TWICE;
     } else {
       // init by dup_tablet_scan_task_.
@@ -397,7 +401,7 @@ int ObDupTableLSHandler::init(bool is_dup_table)
           DUP_TABLE_LOG(WARN, "init ts_sync_mgr failed", K(ret));
         } else if (OB_FAIL(tablets_mgr_ptr_->init(this))) {
           DUP_TABLE_LOG(WARN, "init tablets_mgr failed", K(ret));
-        } else if (ATOMIC_LOAD(&is_master_) && OB_FAIL(leader_takeover_(true /*is_resume*/))) {
+        } else if (ls_state_helper_.is_leader() && OB_FAIL(leader_takeover_(true /*is_resume*/))) {
           DUP_TABLE_LOG(WARN, "leader takeover in init failed", K(ret));
         } else {
           ATOMIC_STORE(&is_inited_, true);
@@ -421,20 +425,146 @@ int ObDupTableLSHandler::init(bool is_dup_table)
         ts_sync_mgr_ptr_ = nullptr;
         tablets_mgr_ptr_ = nullptr;
       }
-      DUP_TABLE_LOG(INFO, "ls handler init", K(ret), K(ls_id_), K(is_master()), K(is_follower()),
+      DUP_TABLE_LOG(INFO, "ls handler init", K(ret), KPC(this));
-                          KPC(lease_mgr_ptr_), KPC(tablets_mgr_ptr_), KP(ts_sync_mgr_ptr_));
     }
   }
   return ret;
 }
 
+void ObDupTableLSHandler::start()
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+
+  ObDupTableLSRoleStateContainer restore_state_container;
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_START_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_START_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  }
+}
+
+void ObDupTableLSHandler::stop()
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+
+  ObDupTableLSRoleStateContainer restore_state_container;
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_STOP_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_STOP_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  }
+}
+
+int ObDupTableLSHandler::offline()
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+
+  ObDupTableLSRoleStateContainer restore_state_container;
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_OFFLINE_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  } else if (OB_NOT_NULL(log_operator_) && log_operator_->is_busy()) {
+    ret = OB_EAGAIN;
+    DUP_TABLE_LOG(WARN, "wait log synced before offline", K(ret), KPC(this));
+  } else if (OB_NOT_NULL(tablets_mgr_ptr_) && OB_FAIL(tablets_mgr_ptr_->offline())) {
+    DUP_TABLE_LOG(WARN, "dup tablets mgr offline failed", K(ret), KPC(this));
+  } else if (OB_NOT_NULL(lease_mgr_ptr_) && OB_FAIL(lease_mgr_ptr_->offline())) {
+    DUP_TABLE_LOG(WARN, "dup lease mgr offline failed", K(ret), KPC(this));
+  } else if (OB_NOT_NULL(ts_sync_mgr_ptr_) && OB_FAIL(ts_sync_mgr_ptr_->offline())) {
+    DUP_TABLE_LOG(WARN, "dup ts mgr offline failed", K(ret), KPC(this));
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_OFFLINE_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  }
+  return ret;
+}
+
+int ObDupTableLSHandler::online()
+{
+  int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+
+  ObDupTableLSRoleStateContainer restore_state_container;
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_ONLINE_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  } else if (OB_FAIL(dup_ls_ckpt_.online())) {
+    DUP_TABLE_LOG(WARN, "dup table checkpoint online failed", K(ret), KPC(this));
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_ONLINE_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  }
+
+  return ret;
+}
+
+int ObDupTableLSHandler::safe_to_destroy(bool &is_dup_table_handler_safe)
+{
+  int ret = OB_SUCCESS;
+  is_dup_table_handler_safe = false;
+  if (OB_NOT_NULL(log_operator_) && log_operator_->is_busy()) {
+    ret = OB_EAGAIN;
+    DUP_TABLE_LOG(WARN, "wait log synced before destroy", K(ret), KPC(this));
+  }
+
+  if (OB_SUCC(ret)) {
+    is_dup_table_handler_safe = true;
+  }
+
+  return ret;
+}
+
 void ObDupTableLSHandler::destroy() { reset(); }
 
+// int ObDupTableLSHandler::offline()
+// {
+//   int ret = OB_SUCCESS;
+//   }
+//   return ret;
+// }
+//
+// int ObDupTableLSHandler::online() {}
+
 void ObDupTableLSHandler::reset()
 {
   // ATOMIC_STORE(&is_inited_, false);
   is_inited_ = false;
-  is_master_ = false;
+  ls_state_helper_.reset();
 
   dup_ls_ckpt_.reset();
 
@@ -470,52 +600,76 @@ void ObDupTableLSHandler::reset()
   }
 }
 
-bool ObDupTableLSHandler::is_master()
+// bool ObDupTableLSHandler::is_master()
-{
+// {
-  bool sub_master = true;
+//   bool sub_master = true;
-  if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
+//   if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
-    sub_master = sub_master && ts_sync_mgr_ptr_->is_master();
+//     sub_master = sub_master && ts_sync_mgr_ptr_->is_master();
-  }
+//   }
-  if (OB_NOT_NULL(lease_mgr_ptr_)) {
+//   if (OB_NOT_NULL(lease_mgr_ptr_)) {
-    sub_master = sub_master && lease_mgr_ptr_->is_master();
+//     sub_master = sub_master && lease_mgr_ptr_->is_master();
-  }
+//   }
-  if (OB_NOT_NULL(tablets_mgr_ptr_)) {
+//   if (OB_NOT_NULL(tablets_mgr_ptr_)) {
-    sub_master = sub_master && tablets_mgr_ptr_->is_master();
+//     sub_master = sub_master && tablets_mgr_ptr_->is_master();
-  }
+//   }
+//
+//   return (ATOMIC_LOAD(&is_master_)) && sub_master;
+// }
 
-  return (ATOMIC_LOAD(&is_master_)) && sub_master;
+// bool ObDupTableLSHandler::is_follower()
+// {
+//   bool sub_not_master = true;
+//   if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
+//     sub_not_master = sub_not_master && !ts_sync_mgr_ptr_->is_master();
+//   }
+//   if (OB_NOT_NULL(lease_mgr_ptr_)) {
+//     sub_not_master = sub_not_master && !lease_mgr_ptr_->is_master();
+//   }
+//   if (OB_NOT_NULL(tablets_mgr_ptr_)) {
+//     sub_not_master = sub_not_master && !tablets_mgr_ptr_->is_master();
+//   }
+//
+//   return (!ATOMIC_LOAD(&is_master_)) && sub_not_master;
+// }
+
+bool ObDupTableLSHandler::is_inited()
+{
+  return ATOMIC_LOAD(&is_inited_);
 }
 
-bool ObDupTableLSHandler::is_follower()
+bool ObDupTableLSHandler::is_master() { return ls_state_helper_.is_leader_serving(); }
-{
-  bool sub_not_master = true;
-  if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
-    sub_not_master = sub_not_master && !ts_sync_mgr_ptr_->is_master();
-  }
-  if (OB_NOT_NULL(lease_mgr_ptr_)) {
-    sub_not_master = sub_not_master && !lease_mgr_ptr_->is_master();
-  }
-  if (OB_NOT_NULL(tablets_mgr_ptr_)) {
-    sub_not_master = sub_not_master && !tablets_mgr_ptr_->is_master();
-  }
 
-  return (!ATOMIC_LOAD(&is_master_)) && sub_not_master;
+// bool ObDupTableLSHandler::is_online()
-}
+// {
+//   bool sub_online =  true;
+//   if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
+//     sub_online = sub_online && !ts_sync_mgr_ptr_->is_master();
+//   }
+//   if (OB_NOT_NULL(lease_mgr_ptr_)) {
+//     sub_online = sub_online && !lease_mgr_ptr_->is_master();
+//   }
+//   if (OB_NOT_NULL(tablets_mgr_ptr_)) {
+//     sub_online = sub_online && !tablets_mgr_ptr_->is_master();
+//   }
+//
+// }
 
 int ObDupTableLSHandler::ls_loop_handle()
 {
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
-  // TODO check stopped
+  if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablets_mgr_ptr_)
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablets_mgr_ptr_)
       || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "dup table ls handle not init", K(ret));
+  } else if (!ls_state_helper_.is_active_ls()) {
+    ret = OB_LS_OFFLINE;
+    DUP_TABLE_LOG(WARN, "the ls is not active", K(ret), KPC(this));
   } else if (!has_dup_tablet()) {
     ret = OB_NO_TABLET;
     DUP_TABLE_LOG(INFO, "no dup tablet, no need to do loop worker", K(ret), KPC(tablets_mgr_ptr_));
   } else {
-    if (is_master()) {
+    if (ls_state_helper_.is_leader()) {
       if (OB_ISNULL(log_operator_) || !log_operator_->is_busy()) {
         // handle lease request and collect follower info
         DupTableTsInfo min_lease_ts_info;
@@ -539,16 +693,20 @@ int ObDupTableLSHandler::ls_loop_handle()
         DUP_TABLE_LOG(WARN, "update all ts info cache failed", K(tmp_ret));
       }
 
-    } else if (is_follower()) {
+    } else if (ls_state_helper_.is_follower()) {
       if (OB_FAIL(lease_mgr_ptr_->follower_handle())) {
         DUP_TABLE_LOG(WARN, "follower lease handle failed", K(ret));
       }
+    } else {
+      DUP_TABLE_LOG(INFO, "undefined role state for dup table ls, skip this loop", K(ret),
+                    K(ls_id_), K(ls_state_helper_));
     }
-    DUP_TABLE_LOG(DEBUG, "loop running : dup table ls handler", K(ret), K(ls_id_), K(is_master()),
+    DUP_TABLE_LOG(DEBUG, "loop running : dup table ls handler", K(ret), K(ls_id_),
-                  KPC(lease_mgr_ptr_), KPC(tablets_mgr_ptr_), KPC(log_operator_));
+                  K(ls_state_helper_), KPC(lease_mgr_ptr_), KPC(tablets_mgr_ptr_),
+                  KPC(log_operator_));
 
     const int64_t fast_cur_time = ObTimeUtility::fast_current_time();
-    const bool is_leader = is_master();
+    const bool is_leader = ls_state_helper_.is_leader();
 
     if (fast_cur_time - last_diag_info_print_us_[DupTableDiagStd::TypeIndex::LEASE_INDEX]
         >= DupTableDiagStd::DUP_DIAG_PRINT_INTERVAL[DupTableDiagStd::TypeIndex::LEASE_INDEX]) {
@@ -585,15 +743,20 @@ int ObDupTableLSHandler::refresh_dup_table_tablet(common::ObTabletID tablet_id,
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(tablets_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(tablets_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     if (is_dup_table) {
-      DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_),
+      DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_), K(is_dup_table),
-                    KP(tablets_mgr_ptr_));
+                    KPC(this));
     }
+  } else if (!ls_state_helper_.is_leader()) {
+    ret = OB_NOT_MASTER;
+    DUP_TABLE_LOG(WARN, "No need to refresh dup table schema", K(ret), K(tablet_id),
+                  K(is_dup_table), KPC(this));
   } else if (OB_FAIL(tablets_mgr_ptr_->refresh_dup_tablet(tablet_id, is_dup_table, refresh_time))) {
     if (ret != OB_NOT_MASTER) {
-      DUP_TABLE_LOG(WARN, "refresh dup table tablet failed", K(ret), K(tablet_id), K(is_dup_table));
+      DUP_TABLE_LOG(WARN, "refresh dup table tablet failed", K(ret), K(tablet_id), K(is_dup_table),
+                    KPC(this));
     } else {
       ret = OB_SUCCESS;
     }
@@ -605,9 +768,12 @@ int ObDupTableLSHandler::refresh_dup_table_tablet(common::ObTabletID tablet_id,
 int ObDupTableLSHandler::recive_lease_request(const ObDupTableLeaseRequest &lease_req)
 {
   int ret = OB_SUCCESS;
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(lease_mgr_ptr_));
+  } else if (!ls_state_helper_.is_leader_serving()) {
+    ret = OB_NOT_MASTER;
+    DUP_TABLE_LOG(WARN, "No need to recive lease request", K(ret), K(lease_req), KPC(this));
   } else if (OB_FAIL(ts_sync_mgr_ptr_->handle_ts_sync_response(lease_req))) {
     DUP_TABLE_LOG(WARN, "handle ts sync response failed", K(ret));
   } else if (OB_FAIL(lease_mgr_ptr_->recive_lease_request(lease_req))) {
@@ -620,9 +786,12 @@ int ObDupTableLSHandler::handle_ts_sync_response(const ObDupTableTsSyncResponse
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
+  } else if (!ls_state_helper_.is_leader_serving()) {
+    ret = OB_NOT_MASTER;
+    DUP_TABLE_LOG(WARN, "No need to handle ts sync response", K(ret), K(ts_sync_resp), KPC(this));
   } else if (OB_FAIL(ts_sync_mgr_ptr_->handle_ts_sync_response(ts_sync_resp))) {
     DUP_TABLE_LOG(WARN, "handle ts sync response failed", K(ret));
   }
@@ -634,9 +803,12 @@ int ObDupTableLSHandler::handle_ts_sync_request(const ObDupTableTsSyncRequest &t
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
+  } else if (!ls_state_helper_.is_follower_serving()) {
+    ret = OB_NOT_FOLLOWER;
+    DUP_TABLE_LOG(WARN, "No need to handle ts sync request", K(ret), K(ts_sync_req), KPC(this));
   } else if (OB_FAIL(ts_sync_mgr_ptr_->handle_ts_sync_request(ts_sync_req))) {
     DUP_TABLE_LOG(WARN, "handle ts sync request failed", K(ret));
   }
@@ -663,7 +835,7 @@ int ObDupTableLSHandler::check_redo_sync_completed(const ObTransID &tx_id,
   before_prepare_gts.set_invalid();
   int64_t start_us = OB_INVALID_TIMESTAMP;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_), KP(lease_mgr_ptr_),
                   KP(ts_sync_mgr_ptr_));
@@ -775,7 +947,7 @@ int ObDupTableLSHandler::block_confirm_with_dup_tablet_change_snapshot(
 
   ATOMIC_INC(&total_block_confirm_ref_);
 
-  if (!ATOMIC_LOAD(&is_inited_)) {
+  if (!is_inited()) {
     // do nothing
     ret = OB_SUCCESS;
   } else {
@@ -788,7 +960,7 @@ int ObDupTableLSHandler::gc_dup_tablets(const int64_t gc_ts, const int64_t max_t
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_)) {
+  if (!is_inited() || !ls_state_helper_.is_leader_serving()) {
     // do nothing
   } else if (OB_ISNULL(tablets_mgr_ptr_)) {
     ret = OB_NOT_INIT;
@@ -808,7 +980,7 @@ int ObDupTableLSHandler::try_to_confirm_tablets_(const share::SCN &confirm_scn)
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(tablets_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(tablets_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "ObDupTableLSHandler not init", K(ret), K(is_inited_),
                   KP(tablets_mgr_ptr_));
@@ -871,6 +1043,9 @@ int ObDupTableLSHandler::check_dup_tablet_readable(const ObTabletID &tablet_id,
   } else if (OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablets_mgr_ptr_)) {
     // no dup tablet in ls
     readable = false;
+  } else if (!ls_state_helper_.is_active_ls()) {
+    readable = false;
+    DUP_TABLE_LOG(INFO, "the ls is not active", K(ret), KPC(this));
   } else if (!has_dup_tablet()) {
     readable = false;
     interface_stat_.dup_table_follower_read_tablet_not_exist_cnt_++;
@@ -905,6 +1080,7 @@ int ObDupTableLSHandler::check_dup_tablet_readable(const ObTabletID &tablet_id,
 
   return ret;
 }
+
 int64_t ObDupTableLSHandler::get_dup_tablet_count()
 {
   int64_t dup_tablet_cnt = 0;
@@ -933,9 +1109,12 @@ int ObDupTableLSHandler::get_local_ts_info(DupTableTsInfo &ts_info)
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
+  } else if (!ls_state_helper_.is_active_ls()) {
+    ret = OB_LS_OFFLINE;
+    DUP_TABLE_LOG(WARN, "the ls is not active", K(ret), KPC(this));
   } else if (OB_FAIL(ts_sync_mgr_ptr_->get_local_ts_info(ts_info))) {
     DUP_TABLE_LOG(WARN, "get local ts sync info failed", K(ret));
   }
@@ -947,9 +1126,13 @@ int ObDupTableLSHandler::get_cache_ts_info(const common::ObAddr &addr, DupTableT
 {
   int ret = OB_SUCCESS;
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_NOT_INIT;
     DUP_TABLE_LOG(WARN, "DupTableLSHandler not init", K(ret), K(is_inited_), KP(ts_sync_mgr_ptr_));
+  } else if (!ls_state_helper_.is_leader_serving()) {
+    ret = OB_NOT_MASTER;
+    DUP_TABLE_LOG(WARN, "we can not get cached ts info from a follower", K(ret), K(ts_info),
+                  KPC(this));
   } else if (OB_FAIL(ts_sync_mgr_ptr_->get_cache_ts_info(addr, ts_info))) {
     DUP_TABLE_LOG(WARN, "get cache ts info failed", K(ret), K(addr), K(ts_info));
   }
@@ -967,7 +1150,7 @@ int ObDupTableLSHandler::replay(const void *buffer,
   const bool no_dup_tablet_before_replay = !has_dup_tablet();
 
   // cover lease list and tablets list
-  if (!ATOMIC_LOAD(&is_inited_) && OB_FAIL(init(true))) {
+  if (!is_inited() && OB_FAIL(init(true))) {
     DUP_TABLE_LOG(WARN, "init dup_ls_handle in replay failed", K(ret));
   } else if (OB_FAIL(prepare_log_operator_())) {
     DUP_TABLE_LOG(WARN, "init dup_table log operator failed", K(ret));
@@ -1011,11 +1194,25 @@ int ObDupTableLSHandler::replay(const void *buffer,
 void ObDupTableLSHandler::switch_to_follower_forcedly()
 {
   int ret = OB_SUCCESS;
-  if (OB_FAIL(leader_revoke_())) {
+  int tmp_ret = OB_SUCCESS;
-    DUP_TABLE_LOG(ERROR, "switch to follower forcedly failed for dup table", K(ret), K(ls_id_),
+
-                  K(is_master()));
+  ObDupTableLSRoleStateContainer restore_state_container;
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_REVOKE_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  } else if (OB_FAIL(leader_revoke_(true /*forcedly*/))) {
+    DUP_TABLE_LOG(ERROR, "switch to follower forcedly failed for dup table", K(ret), KPC(this));
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_REVOKE_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
   }
-  ATOMIC_STORE(&is_master_, false);
 }
 
 int ObDupTableLSHandler::switch_to_follower_gracefully()
@@ -1023,20 +1220,34 @@ int ObDupTableLSHandler::switch_to_follower_gracefully()
   int ret = OB_SUCCESS;
   int tmp_ret = OB_SUCCESS;
 
-  if (OB_FAIL(leader_revoke_())) {
+  ObDupTableLSRoleStateContainer restore_state_container;
-    DUP_TABLE_LOG(WARN, "switch to follower gracefully failed for dup table", K(ret), K(ls_id_),
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_REVOKE_SUCC,
-                  K(is_master()));
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  } else if (OB_FAIL(leader_revoke_(false /*forcedly*/))) {
+    DUP_TABLE_LOG(WARN, "switch to follower gracefully failed for dup table", K(ret), KPC(this));
   }
 
-  ATOMIC_STORE(&is_master_, false);
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_REVOKE_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  } else {
+    tmp_ret = OB_SUCCESS;
 
-  if (OB_FAIL(ret)) {
+    if (OB_TMP_FAIL(leader_takeover_(true /*is_resume*/))) {
-    if (OB_TMP_FAIL(resume_leader())) {
+      DUP_TABLE_LOG(WARN, "resume leader failed", K(ret), K(tmp_ret), KPC(this));
+    }
+    if (OB_SUCCESS != tmp_ret) {
       ret = OB_LS_NEED_REVOKE;
-      DUP_TABLE_LOG(WARN, "resume leader failed, need revoke", K(ret), K(tmp_ret), K(ls_id_));
+      DUP_TABLE_LOG(WARN, "resume leader failed, need revoke", K(ret), K(tmp_ret), KPC(this));
-    } else {
+    } else if (OB_TMP_FAIL(ls_state_helper_.restore_state(ObDupTableLSRoleState::LS_REVOKE_SUCC,
-      DUP_TABLE_LOG(WARN, "resume leader successfully, return error code", K(ret), K(tmp_ret),
+                                                          restore_state_container))) {
-                    K(ls_id_));
+      DUP_TABLE_LOG(ERROR, "restore ls role state error", K(ret), KPC(this));
     }
   }
 
@@ -1046,62 +1257,115 @@ int ObDupTableLSHandler::switch_to_follower_gracefully()
 int ObDupTableLSHandler::resume_leader()
 {
   int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
 
   const bool is_resume = true;
 
-  if (OB_FAIL(leader_takeover_(is_resume))) {
+  ObDupTableLSRoleStateContainer restore_state_container;
-    DUP_TABLE_LOG(WARN, "resume leader failed for dup table", K(ret), K(ls_id_), K(is_master()));
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  } else if (OB_FAIL(leader_takeover_(is_resume))) {
+    DUP_TABLE_LOG(WARN, "resume leader failed for dup table", K(ret), KPC(this));
   }
 
-  ATOMIC_STORE(&is_master_, true);
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  } else {
+    tmp_ret = OB_SUCCESS;
+
+    if (OB_TMP_FAIL(leader_revoke_(true /*forcedly*/))) {
+      DUP_TABLE_LOG(WARN, "leader revoke failed", K(ret), K(tmp_ret), KPC(this));
+    } else if (OB_TMP_FAIL(ls_state_helper_.restore_state(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
+                                                          restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "restore ls role state error", K(ret), KPC(this));
+    }
+  }
   return ret;
 }
 
 int ObDupTableLSHandler::switch_to_leader()
 {
   int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
 
   const bool is_resume = false;
-  if (OB_FAIL(leader_takeover_(is_resume))) {
+
-    DUP_TABLE_LOG(WARN, "switch to leader failed for dup table", K(ret), K(ls_id_), K(is_master()));
+  ObDupTableLSRoleStateContainer restore_state_container;
+  if (OB_FAIL(ls_state_helper_.prepare_state_change(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
+                                                    restore_state_container))) {
+    DUP_TABLE_LOG(WARN, "prepare state change failed", K(ret), KPC(this));
+    if (OB_NO_NEED_UPDATE == ret) {
+      ret = OB_SUCCESS;
+    }
+  } else if (OB_FAIL(leader_takeover_(is_resume))) {
+    DUP_TABLE_LOG(WARN, "leader takeover failed for dup table", K(ret), KPC(this));
+  }
+
+  if (OB_SUCC(ret)) {
+    if (OB_FAIL(ls_state_helper_.state_change_succ(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
+                                                   restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "change ls role state error", K(ret), KPC(this));
+    }
+  } else {
+    tmp_ret = OB_SUCCESS;
+
+    if (OB_TMP_FAIL(leader_revoke_(true /*forcedly*/))) {
+      DUP_TABLE_LOG(WARN, "leader revoke failed", K(ret), K(tmp_ret), KPC(this));
+    } else if (OB_TMP_FAIL(ls_state_helper_.restore_state(ObDupTableLSRoleState::LS_TAKEOVER_SUCC,
+                                                          restore_state_container))) {
+      DUP_TABLE_LOG(ERROR, "restore ls role state error", K(ret), KPC(this));
+    }
   }
-  ATOMIC_STORE(&is_master_, true);
   return ret;
 }
 
-int ObDupTableLSHandler::leader_revoke_()
+int ObDupTableLSHandler::leader_revoke_(const bool is_forcedly)
 {
   int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
 
-  if (!is_master()) {
+  bool is_logging = false;
-    // ret = OB_STATE_NOT_MATCH;
+  if (OB_NOT_NULL(log_operator_)) {
-    DUP_TABLE_LOG(WARN, "ObDupTableLSHandler has already been follower",
+    log_operator_->rlock_for_log();
-                  K(ret), K(ls_id_), K(is_master()));
+    is_logging = log_operator_->check_is_busy_without_lock();
-  } else {
+  }
-    // clean new/old tablet set
+  if (OB_NOT_NULL(tablets_mgr_ptr_) && OB_TMP_FAIL(tablets_mgr_ptr_->leader_revoke(is_logging))) {
-    if (OB_NOT_NULL(log_operator_)) {
+    DUP_TABLE_LOG(WARN, "tablets_mgr switch to follower failed", K(ret), K(tmp_ret), KPC(this));
-      if (OB_NOT_NULL(tablets_mgr_ptr_)
+    if (!is_forcedly) {
-          && OB_FAIL(tablets_mgr_ptr_->leader_revoke(log_operator_->is_busy()))) {
+      ret = tmp_ret;
-        DUP_TABLE_LOG(WARN, "clean unreadable tablet set failed", K(ret));
+    }
+  }
+  if (OB_NOT_NULL(log_operator_)) {
+    log_operator_->unlock_for_log();
+  }
+
+  if (OB_SUCC(ret) && OB_NOT_NULL(ts_sync_mgr_ptr_)) {
+    if (OB_TMP_FAIL(ts_sync_mgr_ptr_->leader_revoke())) {
+      DUP_TABLE_LOG(WARN, "ts_sync_mgr switch to follower failed", K(ret), K(tmp_ret), KPC(this));
+      if (!is_forcedly) {
+        ret = tmp_ret;
       }
     }
+  }
 
-    if (OB_SUCC(ret) && OB_NOT_NULL(ts_sync_mgr_ptr_)) {
+  if (OB_SUCC(ret) && OB_NOT_NULL(lease_mgr_ptr_)) {
-      if (OB_FAIL(ts_sync_mgr_ptr_->leader_revoke())) {
+    if (OB_TMP_FAIL(lease_mgr_ptr_->leader_revoke())) {
-        DUP_TABLE_LOG(WARN, "ts_sync_mgr switch to follower failed", K(ret));
+      DUP_TABLE_LOG(WARN, "lease_mgr switch to follower failed", K(ret), K(tmp_ret), KPC(this));
+      if (!is_forcedly) {
+        ret = tmp_ret;
       }
     }
-
-    if (OB_SUCC(ret) && OB_NOT_NULL(lease_mgr_ptr_)) {
-      if (OB_FAIL(lease_mgr_ptr_->leader_revoke())) {
-        DUP_TABLE_LOG(WARN, "lease_mgr switch to follower failed", K(ret));
-      }
-    }
-
   }
 
   interface_stat_.reset();
-  DUP_TABLE_LOG(INFO, "Leader Revoke", K(ret), K(ls_id_), KPC(log_operator_));
+  DUP_TABLE_LOG(INFO, "Leader Revoke", K(ret), K(is_forcedly), KPC(this));
   return ret;
 }
 
@@ -1109,29 +1373,24 @@ int ObDupTableLSHandler::leader_takeover_(const bool is_resume)
 {
   int ret = OB_SUCCESS;
 
-  if (is_master()) {
+  // clean ts info cache
-    ret = OB_STATE_NOT_MATCH;
+  if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
-    DUP_TABLE_LOG(ERROR, "unexpected ObDupTableLSHandler role", K(ret), K(ls_id_), K(is_master()));
+    ts_sync_mgr_ptr_->leader_takeover();
-  } else {
+  }
-    // clean ts info cache
+  // extend lease_expired_time
-    if (OB_NOT_NULL(ts_sync_mgr_ptr_)) {
+  if (OB_NOT_NULL(lease_mgr_ptr_)) {
-      ts_sync_mgr_ptr_->leader_takeover();
+    lease_mgr_ptr_->leader_takeover(is_resume);
-    }
+  }
-    // extend lease_expired_time
-    if (OB_NOT_NULL(lease_mgr_ptr_)) {
-      lease_mgr_ptr_->leader_takeover(is_resume);
-    }
 
-    if (OB_NOT_NULL(tablets_mgr_ptr_)) {
+  if (OB_NOT_NULL(tablets_mgr_ptr_)) {
-      if (OB_FAIL(tablets_mgr_ptr_->leader_takeover(
+    if (OB_FAIL(tablets_mgr_ptr_->leader_takeover(
-              is_resume, dup_ls_ckpt_.contain_all_readable_on_replica()))) {
+            is_resume, dup_ls_ckpt_.contain_all_readable_on_replica()))) {
-        DUP_TABLE_LOG(WARN, "clean unreadable tablet set failed", K(ret));
+      DUP_TABLE_LOG(WARN, "clean unreadable tablet set failed", K(ret));
-      }
     }
   }
 
   interface_stat_.reset();
-  DUP_TABLE_LOG(INFO, "Leader Takeover", K(ret), K(ls_id_), K(is_resume));
+  DUP_TABLE_LOG(INFO, "Leader Takeover", K(ret), K(is_resume), KPC(this));
   return ret;
 }
 
@@ -1154,6 +1413,40 @@ int ObDupTableLSHandler::prepare_log_operator_()
   return ret;
 }
 
+int ObDupTableLSHandler::set_dup_table_ls_meta(
+    const ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta,
+    bool need_flush_slog)
+{
+  int ret = OB_SUCCESS;
+
+  DUP_TABLE_LOG(INFO, "try to recover dup table ls meta", K(ret), K(need_flush_slog),
+                K(dup_ls_meta), KPC(this));
+
+  if (OB_FAIL(dup_ls_ckpt_.set_dup_ls_meta(dup_ls_meta))) {
+    DUP_TABLE_LOG(WARN, "set dup ls meta failed", K(ret), K(need_flush_slog), K(dup_ls_meta),
+                  KPC(this));
+  } else if (need_flush_slog && OB_FAIL(dup_ls_ckpt_.flush())) {
+    DUP_TABLE_LOG(WARN, "flush slog failed", K(ret), K(need_flush_slog), K(dup_ls_meta), KPC(this));
+  }
+
+  return ret;
+}
+
+int ObDupTableLSHandler::flush(share::SCN &rec)
+{
+  int ret = OB_SUCCESS;
+
+  if (!ls_state_helper_.is_started()) {
+    ret = OB_LS_NOT_EXIST;
+    DUP_TABLE_LOG(WARN, "this ls is not started", K(ret), K(rec), KPC(this));
+
+  } else {
+    ret = dup_ls_ckpt_.flush();
+  }
+
+  return ret;
+}
+
 int ObDupTableLSHandler::check_and_update_max_replayed_scn(const share::SCN &max_replayed_scn)
 {
   int ret = OB_SUCCESS;
@@ -1186,7 +1479,7 @@ int ObDupTableLSHandler::get_min_lease_ts_info_(DupTableTsInfo &min_ts_info)
   LeaseAddrArray lease_valid_array;
   min_ts_info.reset();
 
-  if (!ATOMIC_LOAD(&is_inited_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
+  if (!is_inited() || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
     ret = OB_INVALID_ARGUMENT;
     DUP_TABLE_LOG(WARN, "invalid arguments", K(is_inited_), KP(lease_mgr_ptr_),
                   KP(ts_sync_mgr_ptr_));
@@ -1226,7 +1519,7 @@ int ObDupTableLSHandler::get_lease_mgr_stat(ObDupLSLeaseMgrStatIterator &collect
   FollowerLeaseMgrStatArr collect_arr;
 
   // collect all leader info
-  if (is_master()) {
+  if (ls_state_helper_.is_leader_serving()) {
     if (OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(ts_sync_mgr_ptr_)) {
       ret = OB_NOT_INIT;
       DUP_TABLE_LOG(WARN, "not init", K(ret), KPC(lease_mgr_ptr_), KP(ts_sync_mgr_ptr_));

src/storage/tx/ob_dup_table_util.h

@@ -14,6 +14,7 @@
 // #include "lib/hash/ob_hashset.h"
 #include "logservice/ob_log_base_type.h"
 #include "storage/tx/ob_dup_table_tablets.h"
+#include "storage/tx/ob_dup_table_lease.h"
 #include "storage/tx/ob_trans_define.h"
 #include "storage/tx/ob_dup_table_stat.h"
 
@@ -96,17 +97,20 @@ class ObDupTableLSHandler : public logservice::ObIReplaySubHandler,
 {
 public:
   ObDupTableLSHandler()
-      : lease_mgr_ptr_(nullptr), ts_sync_mgr_ptr_(nullptr), tablets_mgr_ptr_(nullptr),
+      : ls_state_helper_("DupTableLSHandler"), lease_mgr_ptr_(nullptr), ts_sync_mgr_ptr_(nullptr),
-        log_operator_(nullptr)
+        tablets_mgr_ptr_(nullptr), log_operator_(nullptr)
   {
     reset();
   }
   // init by ObDupTabletScanTask or replay
   int init(bool is_dup_table);
 
+  void start();
   void stop();    // TODO  stop submit log before the log handler is invalid.
-  void wait();    // TODO
+  int safe_to_destroy(bool &is_dup_table_handler_safe);    // TODO
   void destroy(); // remove from dup_table_loop_worker
+  int offline();
+  int online();
   void reset();
 
   const share::ObLSID &get_ls_id() { return ls_id_; }
@@ -119,7 +123,18 @@ public:
   }
 
   bool is_master();
-  bool is_follower();
+  // bool is_follower();
+  bool is_inited();
+  // bool is_online();
+
+  TO_STRING_KV(K(ls_id_),
+               K(ls_state_helper_),
+               K(dup_ls_ckpt_),
+               KPC(lease_mgr_ptr_),
+               KPC(tablets_mgr_ptr_),
+               // KPC(ts_sync_mgr_ptr_),
+               KPC(log_operator_),
+               K(interface_stat_));
 
 public:
   int ls_loop_handle();
@@ -163,7 +178,6 @@ public:
                                 bool &readable);
 
 public:
-  bool is_inited() { return is_inited_; }
   int64_t get_dup_tablet_count();
   bool has_dup_tablet();
   int gc_dup_tablets(const int64_t gc_ts, const int64_t max_task_interval);
@@ -185,17 +199,15 @@ public:
   int resume_leader();
   int switch_to_leader();
 
-  int set_dup_table_ls_meta(const ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta)
+  int set_dup_table_ls_meta(const ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta,
-  {
+                            bool need_flush_slog = false);
-    return dup_ls_ckpt_.set_dup_ls_meta(dup_ls_meta);
-  }
   int get_dup_table_ls_meta(ObDupTableLSCheckpoint::ObLSDupTableMeta &dup_ls_meta) const
   {
     return dup_ls_ckpt_.get_dup_ls_meta(dup_ls_meta);
   }
 
   share::SCN get_rec_scn() { return dup_ls_ckpt_.get_lease_log_rec_scn(); }
-  int flush(share::SCN &rec) { return dup_ls_ckpt_.flush(); }
+  int flush(share::SCN &rec);
 
   logservice::ObLogHandler *get_log_handler() { return log_handler_; }
 
@@ -207,22 +219,24 @@ public:
   int64_t get_total_block_confirm_ref() { return ATOMIC_LOAD(&total_block_confirm_ref_); }
 
   int check_and_update_max_replayed_scn(const share::SCN &max_replayed_scn);
+
 private:
   DISALLOW_COPY_AND_ASSIGN(ObDupTableLSHandler);
 
   int prepare_log_operator_();
   int get_min_lease_ts_info_(DupTableTsInfo &min_ts_info);
   int leader_takeover_(const bool is_resume);
-  int leader_revoke_();
+  int leader_revoke_(const bool is_forcedly);
 
   int try_to_confirm_tablets_(const share::SCN &confirm_ts);
 
 private:
   share::ObLSID ls_id_;
-  bool is_master_;  // set by role change
+
-  bool is_inited_;  // ste by replay or dup_tablet_scan_task_
+  bool is_inited_;
-  bool is_stopped_; // TODO
+
-  // TODO: is_stopped_;
+  // set these flag for a normal ls without dup_table
+  ObDupTableLSRoleStateHelper ls_state_helper_;
 
   int64_t total_block_confirm_ref_; // block new dup tablet confirmed