fix change member list fetch ls transfer scn timeout issue

src/storage/high_availability/ob_ls_member_list_service.cpp

@@ -24,6 +24,7 @@ namespace storage
 ObLSMemberListService::ObLSMemberListService()
   : is_inited_(false),
     ls_(NULL),
+    transfer_scn_iter_lock_(),
     log_handler_(NULL)
 {
 }
@@ -166,6 +167,65 @@ int ObLSMemberListService::switch_learner_to_acceptor(
   return ret;
 }
 
+int ObLSMemberListService::get_max_tablet_transfer_scn(share::SCN &transfer_scn)
+{
+  int ret = OB_SUCCESS;
+  const bool need_initial_state = false;
+  ObHALSTabletIDIterator iter(ls_->get_ls_id(), need_initial_state);
+  share::SCN max_transfer_scn = share::SCN::min_scn();
+  static const int64_t LOCK_TIMEOUT = 100_ms; // 100ms
+  if (OB_UNLIKELY(!is_inited_)) {
+    ret = OB_NOT_INIT;
+    STORAGE_LOG(WARN, "not inited", K(ret), K_(is_inited));
+  } else if (OB_FAIL(ls_->build_tablet_iter(iter))) {
+    STORAGE_LOG(WARN, "failed to build tablet iter", K(ret));
+  } else if (OB_FAIL(transfer_scn_iter_lock_.lock(LOCK_TIMEOUT))) {
+    STORAGE_LOG(WARN, "failed to lock transfer scn iter lock", K(ret));
+  } else {
+    ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
+    common::ObTabletID tablet_id;
+    ObTabletMapKey key;
+    key.ls_id_ = ls_->get_ls_id();
+    ObTabletCreateDeleteMdsUserData mds_data;
+    bool committed_flag = false;
+    ObTabletHandle tablet_handle;
+    const WashTabletPriority priority = WashTabletPriority::WTP_LOW;
+    while (OB_SUCC(ret)) {
+      mds_data.reset();
+      if (OB_FAIL(iter.get_next_tablet_id(tablet_id))) {
+        if (OB_ITER_END == ret) {
+          ret = OB_SUCCESS;
+          break;
+        } else {
+          STORAGE_LOG(WARN, "failed to get tablet id", K(ret));
+        }
+      } else if (OB_FALSE_IT(key.tablet_id_ = tablet_id)) {
+      } else if (OB_FAIL(t3m->get_tablet(priority, key, tablet_handle))) {
+        STORAGE_LOG(WARN, "failed to get tablet", K(ret), K(key));
+      } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_latest_tablet_status(mds_data, committed_flag))) {
+        if (OB_EMPTY_RESULT == ret || OB_ERR_SHARED_LOCK_CONFLICT == ret) {
+          STORAGE_LOG(INFO, "committed tablet_status does not exist", K(ret), K(key));
+          ret = OB_SUCCESS;
+        } else {
+          STORAGE_LOG(WARN, "failed to get mds table", KR(ret), K(key));
+        }
+      } else if (!committed_flag) {
+        continue;
+      } else if (share::SCN::invalid_scn() == mds_data.transfer_scn_) {
+        // do nothing
+      } else {
+        transfer_scn = mds_data.transfer_scn_;
+        max_transfer_scn = MAX(transfer_scn, max_transfer_scn);
+      }
+    }
+    if (OB_SUCC(ret)) {
+      transfer_scn = max_transfer_scn;
+    }
+    transfer_scn_iter_lock_.unlock();
+  }
+  return ret;
+}
+
 int ObLSMemberListService::get_leader_config_version_and_transfer_scn_(
     palf::LogConfigVersion &leader_config_version,
     share::SCN &leader_transfer_scn)

src/storage/high_availability/ob_ls_member_list_service.h

@@ -46,6 +46,7 @@ public:
   int replace_member_with_learner(const common::ObMember &added_member,
                                   const common::ObMember &removed_member,
                                   const int64_t timeout);
+  int get_max_tablet_transfer_scn(share::SCN &transfer_scn);
 
 private:
   int get_leader_config_version_and_transfer_scn_(
@@ -64,6 +65,7 @@ private:
 private:
   bool is_inited_;
   storage::ObLS *ls_;
+  lib::ObMutex transfer_scn_iter_lock_;
   logservice::ObLogHandler *log_handler_;
   DISALLOW_COPY_AND_ASSIGN(ObLSMemberListService);
 };

src/storage/ls/ob_ls.cpp

@@ -1852,7 +1852,7 @@ int ObLS::get_transfer_scn(share::SCN &scn)
   } else if (OB_UNLIKELY(is_stopped_)) {
     ret = OB_NOT_RUNNING;
     LOG_WARN("ls stopped", K(ret), K_(ls_meta));
-  } else if (OB_FAIL(ls_tablet_svr_.get_max_tablet_transfer_scn(max_tablet_scn))) {
+  } else if (OB_FAIL(member_list_service_.get_max_tablet_transfer_scn(max_tablet_scn))) {
     LOG_WARN("failed to get max tablet transfer scn", K(ret), K_(ls_meta));
   } else {
     scn = MAX(max_tablet_scn, ls_meta_.get_transfer_scn());

src/storage/ls/ob_ls.h

@@ -509,7 +509,6 @@ public:
   DELEGATE_WITH_RET(ls_tablet_svr_, flush_mds_table, int);
   DELEGATE_WITH_RET(ls_tablet_svr_, enable_to_read, void);
   DELEGATE_WITH_RET(ls_tablet_svr_, disable_to_read, void);
-  DELEGATE_WITH_RET(ls_tablet_svr_, get_max_tablet_transfer_scn, int);
   DELEGATE_WITH_RET(ls_tablet_svr_, get_tablet_with_timeout, int);
   DELEGATE_WITH_RET(ls_tablet_svr_, get_mds_table_mgr, int);
 
@@ -657,6 +656,7 @@ public:
   DELEGATE_WITH_RET(member_list_service_, replace_member, int);
   DELEGATE_WITH_RET(member_list_service_, replace_member_with_learner, int);
   DELEGATE_WITH_RET(member_list_service_, get_config_version_and_transfer_scn, int);
+  DELEGATE_WITH_RET(member_list_service_, get_max_tablet_transfer_scn, int);
   DELEGATE_WITH_RET(log_handler_, add_learner, int);
   DELEGATE_WITH_RET(log_handler_, replace_learners, int);
   DELEGATE_WITH_RET(block_tx_service_, ha_block_tx, int);

src/storage/ls/ob_ls_tablet_service.cpp

@@ -6053,59 +6053,6 @@ int ObLSTabletService::flush_mds_table(int64_t recycle_scn)
   return ret;
 }
 
-int ObLSTabletService::get_max_tablet_transfer_scn(share::SCN &transfer_scn)
-{
-  int ret = OB_SUCCESS;
-  const bool need_initial_state = false;
-  ObHALSTabletIDIterator iter(ls_->get_ls_id(), need_initial_state);
-  share::SCN max_transfer_scn = share::SCN::min_scn();
-  if (OB_UNLIKELY(!is_inited_)) {
-    ret = OB_NOT_INIT;
-    LOG_WARN("not inited", K(ret), K_(is_inited));
-  } else if (OB_FAIL(build_tablet_iter(iter))) {
-    LOG_WARN("failed to build tablet iter", K(ret), KPC(this));
-  } else {
-    ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
-    common::ObTabletID tablet_id;
-    ObTabletMapKey key;
-    key.ls_id_ = ls_->get_ls_id();
-    ObTabletCreateDeleteMdsUserData mds_data;
-    ObTabletHandle tablet_handle;
-    const WashTabletPriority priority = WashTabletPriority::WTP_LOW;
-    while (OB_SUCC(ret)) {
-      mds_data.reset();
-      if (OB_FAIL(iter.get_next_tablet_id(tablet_id))) {
-        if (OB_ITER_END == ret) {
-          ret = OB_SUCCESS;
-          break;
-        } else {
-          LOG_WARN("failed to get tablet id", K(ret));
-        }
-      } else if (OB_FALSE_IT(key.tablet_id_ = tablet_id)) {
-      } else if (OB_FAIL(t3m->get_tablet(priority, key, tablet_handle))) {
-          LOG_WARN("failed to get tablet", K(ret), K(key));
-      } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_tablet_status(
-          share::SCN::max_scn(), mds_data, ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US))) {
-        if (OB_EMPTY_RESULT == ret || OB_ERR_SHARED_LOCK_CONFLICT == ret) {
-          LOG_INFO("committed tablet_status does not exist", K(ret), K(key));
-          ret = OB_SUCCESS;
-        } else {
-          LOG_WARN("failed to get mds table", KR(ret), K(key));
-        }
-      } else if (share::SCN::invalid_scn() == mds_data.transfer_scn_) {
-        // do nothing
-      } else {
-        transfer_scn = mds_data.transfer_scn_;
-        max_transfer_scn = MAX(transfer_scn, max_transfer_scn);
-      }
-    }
-    if (OB_SUCC(ret)) {
-      transfer_scn = max_transfer_scn;
-    }
-  }
-  return ret;
-}
-
 int ObLSTabletService::set_frozen_for_all_memtables()
 {
   int ret = OB_SUCCESS;

src/storage/ls/ob_ls_tablet_service.h

@@ -412,7 +412,6 @@ public:
   int get_all_tablet_ids(const bool except_ls_inner_tablet, common::ObIArray<ObTabletID> &tablet_id_array);
 
   int flush_mds_table(int64_t recycle_scn);
-  int get_max_tablet_transfer_scn(share::SCN &transfer_scn);
 protected:
   virtual int prepare_dml_running_ctx(
       const common::ObIArray<uint64_t> *column_ids,