[MDS] reset mds table state when offline

src/storage/ls/ob_ls_tablet_service.cpp

@@ -5923,7 +5923,7 @@ int ObLSTabletService::DestroyMemtableAndMemberAndMdsTableOperator::operator()(c
     }
   } else if (OB_FAIL(handle.get_obj()->release_memtables())) {
     LOG_WARN("failed to release memtables", K(tenant_id), K(tablet_id));
-  } else if (OB_FAIL(handle.get_obj()->forcely_release_all_mds_nodes("OFFLINE"))) {
+  } else if (OB_FAIL(handle.get_obj()->forcely_reset_mds_table("OFFLINE"))) {
     LOG_WARN("failed to release mds_table", K(tenant_id), K(tablet_id));
   } else if (!tablet_id.is_ls_inner_tablet() && OB_FAIL(handle.get_obj()->reset_storage_related_member())) {
     LOG_WARN("failed to destroy storage related member", K(ret), K(tenant_id), K(tablet_id));

src/storage/multi_data_source/mds_table_base.h

@@ -150,7 +150,7 @@ public:
   virtual ObTabletID get_tablet_id() const;
   virtual bool is_flushing() const;
   virtual int fill_virtual_info(ObIArray<MdsNodeInfoForVirtualTable> &mds_node_info_array) const = 0;
-  virtual int forcely_release_all_mds_nodes(const char *reason) = 0;
+  virtual int forcely_reset_mds_table(const char *reason) = 0;
   void mark_removed_from_t3m(ObTabletPointer *pointer);// need called in del tablet phase
   bool is_removed_from_t3m() const;
   int64_t get_removed_from_t3m_ts() const;
@@ -260,14 +260,16 @@ protected:
     : do_init_tablet_pointer_(nullptr),
     do_remove_tablet_pointer_(nullptr),
     init_ts_(0),
+    last_reset_ts_(0),
     remove_ts_(0),
     init_trace_id_(),
     remove_trace_id_() {}
-    TO_STRING_KV(KP_(do_init_tablet_pointer), KP_(do_remove_tablet_pointer), KTIME_(init_ts), KTIME_(remove_ts),
+    TO_STRING_KV(KP_(do_init_tablet_pointer), KP_(do_remove_tablet_pointer), KTIME_(init_ts), KTIME_(last_reset_ts),
-                 K_(init_trace_id), K_(remove_trace_id));
+                 KTIME_(remove_ts), K_(init_trace_id), K_(remove_trace_id));
     ObTabletPointer *do_init_tablet_pointer_;// can not be accessed, jsut record it to debug
     ObTabletPointer *do_remove_tablet_pointer_;// can not be accessed, jsut record it to debug
     int64_t init_ts_;
+    int64_t last_reset_ts_;
     int64_t remove_ts_;
     ObCurTraceId::TraceId init_trace_id_;
     ObCurTraceId::TraceId remove_trace_id_;

src/storage/multi_data_source/mds_table_handle.h

@@ -45,7 +45,7 @@ public:
   int fill_virtual_info(ObIArray<MdsNodeInfoForVirtualTable> &mds_node_info_array) const;
   int mark_removed_from_t3m(ObTabletPointer *pointer);
   template <int N>
-  int forcely_release_all_mds_nodes(const char (&reason)[N]);
+  int forcely_reset_mds_table(const char (&reason)[N]);
   /******************************Single Key Unit Access Interface**********************************/
   template <typename T>
   int set(T &&data, MdsCtx &ctx, const int64_t lock_timeout_us = 0);

src/storage/multi_data_source/mds_table_handle.ipp

@@ -698,7 +698,7 @@ inline int MdsTableHandle::mark_removed_from_t3m(ObTabletPointer *pointer)
 }
 
 template <int N>
-inline int MdsTableHandle::forcely_release_all_mds_nodes(const char (&reason)[N])
+inline int MdsTableHandle::forcely_reset_mds_table(const char (&reason)[N])
 {
   int ret = OB_SUCCESS;
   CHECK_MDS_TABLE_INIT();
@@ -706,7 +706,7 @@ inline int MdsTableHandle::forcely_release_all_mds_nodes(const char (&reason)[N]
     ret = OB_BAD_NULL_ERROR;
     MDS_LOG(WARN, "p_mds_table_base_ is invalid", K(*this));
   } else {
-    p_mds_table_base_->forcely_release_all_mds_nodes(reason);
+    p_mds_table_base_->forcely_reset_mds_table(reason);
   }
   return ret;
 }

src/storage/multi_data_source/mds_table_impl.h

@@ -146,7 +146,7 @@ public:
     ForEachUnitFillVirtualInfoHelper helper(mds_node_info_array);
     return unit_tuple_.for_each(helper);
   }
-  virtual int forcely_release_all_mds_nodes(const char *reason) override;
+  virtual int forcely_reset_mds_table(const char *reason) override;
   /*****************************Single Key Unit Access Interface***********************************/
   template <typename T>
   int set(T &&data,

src/storage/multi_data_source/mds_table_impl.ipp

@@ -1064,7 +1064,11 @@ void MdsTableImpl<MdsTableType>::on_flush_(const share::SCN &flush_scn, const in
     MDS_LOG_FLUSH(WARN, "flush failed");
     flushing_scn_.reset();
   } else if (!flushing_scn_.is_valid() || flushing_scn_ != flush_scn) {
-    MDS_LOG_FLUSH(ERROR, "flush version mismatch!");
+    if (rec_scn_ == share::SCN::max_scn()) {
+      MDS_LOG_FLUSH(WARN, "maybe meet concurrent reset mds table");
+    } else {
+      MDS_LOG_FLUSH(ERROR, "flush version mismatch!");
+    }
   } else {
     last_flushed_scn_ = flushing_scn_;
     flushing_scn_.reset();
@@ -1437,7 +1441,7 @@ struct ForcelyReleaseAllNodeOp
   const char *reason_;
 };
 template <typename MdsTableType>
-int MdsTableImpl<MdsTableType>::forcely_release_all_mds_nodes(const char *reason)
+int MdsTableImpl<MdsTableType>::forcely_reset_mds_table(const char *reason)
 {
   #define PRINT_WRAPPER KR(ret), K(*this), K(reason)
   int ret = OB_SUCCESS;
@@ -1445,8 +1449,14 @@ int MdsTableImpl<MdsTableType>::forcely_release_all_mds_nodes(const char *reason
   MdsWLockGuard lg(lock_);
   ForcelyReleaseAllNodeOp op(reason);
   if (OB_FAIL(for_each_scan_row(op))) {
-    MDS_LOG_GC(ERROR, "fail to do recycle");
+    MDS_LOG_GC(ERROR, "fail to do reset");
   } else {
+    debug_info_.last_reset_ts_ = ObClockGenerator::getCurrentTime();
+    flushing_scn_.reset();
+    last_flushed_scn_ = share::SCN::min_scn();
+    last_inner_recycled_scn_ = share::SCN::min_scn();
+    rec_scn_ = share::SCN::max_scn();
+    ATOMIC_STORE(&total_node_cnt_, 0);
     MDS_LOG_GC(INFO, "forcely release all mds nodes");
   }
   return ret;

src/storage/tablet/ob_i_tablet_mds_interface.h

@@ -42,7 +42,7 @@ public:
   int remove(const Key &key, mds::MdsCtx &ctx, const int64_t lock_timeout_us = 0);
   // sometimes mds ndoes needed be forcely released, e.g.: ls offline
   template <int N>
-  int forcely_release_all_mds_nodes(const char (&reason)[N]);// reason must be cimpile-time str
+  int forcely_reset_mds_table(const char (&reason)[N]);// reason must be cimpile-time str
   template <typename T>
   int is_locked_by_others(bool &is_locked, const mds::MdsWriter &self = mds::MdsWriter()) const;
 

src/storage/tablet/ob_i_tablet_mds_interface.ipp

@@ -137,7 +137,7 @@ inline int ObITabletMdsInterface::check_mds_written(bool &written)
 }
 
 template <int N>
-int ObITabletMdsInterface::forcely_release_all_mds_nodes(const char (&reason)[N])
+int ObITabletMdsInterface::forcely_reset_mds_table(const char (&reason)[N])
 {
   #define PRINT_WRAPPER KR(ret), K(reason)
   MDS_TG(10_ms);
@@ -155,7 +155,7 @@ int ObITabletMdsInterface::forcely_release_all_mds_nodes(const char (&reason)[N]
   } else if (!handle.is_valid()) {
     ret = OB_ERR_UNEXPECTED;
     MDS_LOG_GC(WARN, "mds cannot be NULL");
-  } else if (CLICK_FAIL(handle.forcely_release_all_mds_nodes(reason))) {
+  } else if (CLICK_FAIL(handle.forcely_reset_mds_table(reason))) {
     MDS_LOG_GC(WARN, "fail to release mds nodes in mds table");
   }
   return ret;