add time guard for data_checkpoint lock add lock for update_clog_checkpoint

src/storage/checkpoint/ob_checkpoint_executor.cpp

@@ -134,13 +134,13 @@ inline void get_min_rec_scn_service_type_by_index_(int index, char* service_type
 int ObCheckpointExecutor::update_clog_checkpoint()
 {
   int ret = OB_SUCCESS;
-  //avoid checkpoint concurrently
+  RLockGuard guard(rwlock_);
-  WLockGuard guard(rwlock_);
   if (update_checkpoint_enabled_) {
     ObFreezer *freezer = ls_->get_freezer();
     if (OB_NOT_NULL(freezer)) {
       SCN checkpoint_scn;
       checkpoint_scn.set_max();
+      WLockGuard guard_for_update_clog_checkpoint(rwlock_for_update_clog_checkpoint_);
       if (OB_FAIL(freezer->get_max_consequent_callbacked_scn(checkpoint_scn))) {
         STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret), K(freezer->get_ls_id()));
       } else {

src/storage/checkpoint/ob_checkpoint_executor.h

@@ -102,6 +102,7 @@ private:
   typedef common::SpinRLockGuard  RLockGuard;
   typedef common::SpinWLockGuard  WLockGuard;
   RWLock rwlock_;
+  RWLock rwlock_for_update_clog_checkpoint_;
 
   bool update_checkpoint_enabled_;
 };

src/storage/checkpoint/ob_data_checkpoint.cpp

@@ -207,7 +207,7 @@ int ObDataCheckpoint::safe_to_destroy(bool &is_safe_destroy)
   }
 
   ObSpinLockGuard ls_frozen_list_guard(ls_frozen_list_lock_);
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] safe_to_destroy");
   new_create_list_.reset();
   ls_frozen_list_.reset();
   active_list_.reset();
@@ -223,7 +223,7 @@ int ObDataCheckpoint::safe_to_destroy(bool &is_safe_destroy)
 SCN ObDataCheckpoint::get_rec_scn()
 {
   ObSpinLockGuard ls_frozen_list_guard(ls_frozen_list_lock_);
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] get_rec_scn");
   int ret = OB_SUCCESS;
   SCN min_rec_scn = SCN::max_scn();
   SCN tmp = SCN::max_scn();
@@ -346,7 +346,7 @@ void ObDataCheckpoint::road_to_flush(SCN rec_scn)
     // active_list -> ls_frozen_list
     ObFreezeCheckpoint *last = nullptr;
     {
-      ObSpinLockGuard guard(lock_);
+      ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] road_to_flush");
       last = active_list_.get_first_greater(rec_scn);
     }
     pop_range_to_ls_frozen_(last, active_list_);
@@ -362,7 +362,7 @@ void ObDataCheckpoint::road_to_flush(SCN rec_scn)
 
 void ObDataCheckpoint::pop_range_to_ls_frozen_(ObFreezeCheckpoint *last, ObCheckpointDList &list)
 {
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] pop_range_to_ls_frozen");
   ObFreezeCheckpoint *cur = list.get_header()->get_next();
   ObFreezeCheckpoint *next = nullptr;
   while (cur != last) {
@@ -402,7 +402,7 @@ void ObDataCheckpoint::ls_frozen_to_active_(int64_t &last_time)
           int ret = OB_SUCCESS;
           auto ob_freeze_checkpoint = iterator.get_next();
           if (ob_freeze_checkpoint->is_active_checkpoint()) {
-            ObSpinLockGuard guard(lock_);
+            ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] ls_frozen_to_new_created_");
             // avoid new active ob_freeze_checkpoint block minor merge
             // push back to new_create_list and wait next freeze
             if(OB_FAIL(transfer_from_ls_frozen_to_new_created_(ob_freeze_checkpoint))) {
@@ -410,7 +410,7 @@ void ObDataCheckpoint::ls_frozen_to_active_(int64_t &last_time)
                           K(ret), K(*ob_freeze_checkpoint));
             }
           } else {
-            ObSpinLockGuard guard(lock_);
+            ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] ls_frozen_to_active_");
             if (OB_FAIL(ob_freeze_checkpoint->check_can_move_to_active(true))) {
               STORAGE_LOG(WARN, "check can freeze failed", K(ret), K(*ob_freeze_checkpoint));
             }
@@ -470,7 +470,7 @@ void ObDataCheckpoint::ls_frozen_to_prepare_(int64_t &last_time)
           } else if (ob_freeze_checkpoint->is_active_checkpoint()) {
             // avoid active ob_freeze_checkpoint block minor merge
             // push back to active_list and wait next freeze
-            ObSpinLockGuard guard(lock_);
+            ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] transfer_from_ls_frozen_to_active_");
             if(OB_SUCCESS != (tmp_ret = (transfer_from_ls_frozen_to_active_(ob_freeze_checkpoint)))) {
               STORAGE_LOG(WARN, "active ob_freeze_checkpoint move to active_list failed",
                           K(tmp_ret), K(*ob_freeze_checkpoint));
@@ -506,7 +506,7 @@ void ObDataCheckpoint::ls_frozen_to_prepare_(int64_t &last_time)
 int ObDataCheckpoint::check_can_move_to_active_in_newcreate()
 {
   int ret = OB_SUCCESS;
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] check_can_move_to_active_in_newcreate");
   if (!ls_freeze_finished_) {
     STORAGE_LOG(INFO, "skip check_can_move when ls freeze");
   } else {
@@ -526,7 +526,7 @@ int ObDataCheckpoint::check_can_move_to_active_in_newcreate()
 
 int ObDataCheckpoint::add_to_new_create(ObFreezeCheckpoint *ob_freeze_checkpoint)
 {
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] add_to_new_create");
   int ret = OB_SUCCESS;
 
   if (OB_FAIL(insert_(ob_freeze_checkpoint, new_create_list_, false))) {
@@ -566,7 +566,7 @@ int ObDataCheckpoint::decide_freeze_clock_(ObFreezeCheckpoint *ob_freeze_checkpo
 
 int ObDataCheckpoint::unlink_from_prepare(ObFreezeCheckpoint *ob_freeze_checkpoint)
 {
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] unlink_from_prepare");
   int ret = OB_SUCCESS;
   // for the node not in prepare_list
   // return OB_SUCCESS
@@ -587,7 +587,7 @@ int ObDataCheckpoint::get_freezecheckpoint_info(
   int ret = OB_SUCCESS;
   freeze_checkpoint_array.reset();
   ObSpinLockGuard ls_frozen_list_guard(ls_frozen_list_lock_);
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] get_freezecheckpoint_info");
   if (OB_FAIL(new_create_list_.get_freezecheckpoint_info(
     freeze_checkpoint_array))) {
     STORAGE_LOG(ERROR, "iterator new_create_list fail",
@@ -620,7 +620,7 @@ int ObDataCheckpoint::traversal_flush_()
   ObSEArray<ObTableHandleV2, 16> flush_tasks;
 
   {
-    ObSpinLockGuard guard(lock_);
+    ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] traversal_flush_");
     if (prepare_list_.is_empty()) {
       STORAGE_LOG(TRACE, "skip traversal_flush", K(ls_freeze_finished_),
                   K(prepare_list_.is_empty()), K(ls_->get_ls_id()));
@@ -749,7 +749,7 @@ int ObDataCheckpoint::get_need_flush_tablets_(const share::SCN recycle_scn,
                                               ObIArray<ObTabletID> &flush_tablets)
 {
   int ret = OB_SUCCESS;
-  ObSpinLockGuard guard(lock_);
+  ObSpinLockTimeGuard guard(lock_, "[data_checkpoint] get_need_flush_tablets_");
   ObSArray<ObFreezeCheckpoint*> need_freeze_checkpoints;
   if (OB_FAIL(new_create_list_.get_need_freeze_checkpoints(
     recycle_scn, need_freeze_checkpoints))) {

src/storage/checkpoint/ob_data_checkpoint.h

@@ -67,6 +67,22 @@ private:
   ObCheckpointDList *dlist_;
 };
 
+class ObSpinLockTimeGuard
+{
+public:
+  ObSpinLockTimeGuard(common::ObSpinLock &lock,
+                      const char *owner = "unknown",
+                      const int64_t warn_threshold = 100 * 1000 /* 100 ms */)
+  : time_guard_(owner, warn_threshold),
+    lock_guard_(lock){}
+
+  ~ObSpinLockTimeGuard() {}
+  void click(const char *mod = NULL) { time_guard_.click(mod); }
+private:
+  ObTimeGuard time_guard_;
+  ObSpinLockGuard lock_guard_;
+};
+
 // responsible for maintenance transaction checkpoint unit
 class ObDataCheckpoint : public ObCommonCheckpoint
 {