update the ckpt scn of dup_table readable set after tablet_log_sync

src/storage/tx/ob_dup_table_base.cpp

@@ -160,7 +160,7 @@ int ObDupTableLSCheckpoint::update_ckpt_after_lease_log_synced(
     if (!start_replay_scn_.is_valid()) {
       start_replay_scn_ = scn;
       DUP_TABLE_LOG(INFO, "[CKPT] replay the first dup_table log", K(ret), KPC(this), K(scn),
-                    K(for_replay));
+                    K(for_replay), K(modify_readable_sets), K(contain_all_readable));
     }
   }
 
@@ -173,7 +173,7 @@ int ObDupTableLSCheckpoint::update_ckpt_after_lease_log_synced(
 
       if (!lease_log_rec_scn_.is_valid()) {
         DUP_TABLE_LOG(INFO, "[CKPT] set rec log scn for lease", K(ret), KPC(this), K(scn),
-                      K(for_replay));
+                      K(for_replay), K(modify_readable_sets), K(contain_all_readable));
         lease_log_rec_scn_ = scn;
       }
     }
@@ -186,6 +186,8 @@ int ObDupTableLSCheckpoint::update_ckpt_after_lease_log_synced(
     } else if (!dup_ls_meta_.readable_tablets_min_base_applied_scn_.is_valid()) {
       dup_ls_meta_.readable_tablets_min_base_applied_scn_ = scn;
     }
+    DUP_TABLE_LOG(INFO, "[CKPT] modify ckpt scn for readable tablets", K(ret), KPC(this), K(scn),
+                  K(for_replay), K(modify_readable_sets), K(contain_all_readable));
   }
 
   return ret;
@@ -502,7 +504,7 @@ int ObDupTableLogOperator::sync_log_succ_(const bool for_replay)
                     K(logging_lease_addrs_));
     } else if (OB_FAIL(tablet_mgr_ptr_->tablet_log_synced(
                    true /*sync_result*/, logging_scn_, for_replay /*for_replay*/,
-                   logging_tablet_set_ids_, contain_all_readable))) {
+                   logging_tablet_set_ids_, modify_readable))) {
       DUP_TABLE_LOG(WARN, "apply tablet_log failed", K(ret), K(logging_scn_),
                     K(logging_tablet_set_ids_));
     } else if (OB_FAIL(dup_ls_ckpt_->update_ckpt_after_lease_log_synced(

src/storage/tx/ob_dup_table_base.h

@@ -647,7 +647,7 @@ public:
   {
     set_special_op_(DUP_SPECIAL_OP_CLEAN_ALL_READABLE_SET);
   }
-  bool need_clean_all_readable_set()
+  bool need_clean_all_readable_set() const
   {
     return DUP_SPECIAL_OP_CLEAN_ALL_READABLE_SET == sp_op_type_;
   }

src/storage/tx/ob_dup_table_lease.cpp

@@ -299,7 +299,7 @@ int ObDupTableLSLeaseMgr::lease_log_synced(const bool sync_result,
   }
 
   if (lease_header_array.count() > 0) {
-    DUP_TABLE_LOG(INFO, "lease log sync", K(ret), K(sync_result), K(for_replay), K(lease_log_scn),
+    DUP_TABLE_LOG(DEBUG, "lease log sync", K(ret), K(sync_result), K(for_replay), K(lease_log_scn),
                   K(lease_header_array), K(is_master()));
   }
   return ret;

src/storage/tx/ob_dup_table_tablets.cpp

@@ -1206,11 +1206,13 @@ int ObLSDupTabletsMgr::tablet_log_synced(const bool sync_result,
                                          const share::SCN &scn,
                                          const bool for_replay,
                                          const DupTabletSetIDArray &unique_id_array,
-                                         bool &merge_confirmed)
+                                         bool &modify_readable_set)
 {
   int ret = OB_SUCCESS;
 
-  merge_confirmed = false;
+  bool clean_readable = false;
+
+  modify_readable_set = false;
   SpinWLockGuard guard(dup_tablets_lock_);
 
   for (int i = 0; OB_SUCC(ret) && i < unique_id_array.count(); i++) {
@@ -1221,19 +1223,36 @@ int ObLSDupTabletsMgr::tablet_log_synced(const bool sync_result,
       ret = OB_INVALID_ARGUMENT;
       DUP_TABLE_LOG(WARN, "invalid common header", K(ret), K(logging_common_header));
     } else if (OB_FAIL(get_target_tablet_set_(logging_common_header, logging_tablet_set))) {
-      DUP_TABLE_LOG(WARN, "get target tablet set failed", K(ret), KPC(logging_tablet_set),
+      if (clean_readable && OB_ENTRY_NOT_EXIST == ret && logging_common_header.is_readable_set()) {
-                    K(logging_common_header));
+        DUP_TABLE_LOG(INFO, "the tablet set has been removed in this log", K(ret),
+                      KPC(logging_tablet_set), K(logging_common_header), K(scn), K(sync_result));
+        ret = OB_SUCCESS;
+      } else {
+        DUP_TABLE_LOG(WARN, "get target tablet set failed", K(ret), KPC(logging_tablet_set),
+                      K(logging_common_header), K(scn), K(sync_result));
+      }
     } else if (logging_common_header.is_readable_set()) {
       // try return empty readable set
-      if (OB_FAIL(check_and_recycle_empty_readable_set(logging_tablet_set))) {
+      bool need_remove = false;
+      if (OB_FAIL(check_and_recycle_empty_readable_set(logging_tablet_set, need_remove))) {
         DUP_TABLE_LOG(WARN, "try return empty readable tablet set", K(ret),
                       KPC(logging_tablet_set));
       }
+      if (need_remove) {
+        modify_readable_set = true;
+      }
     } else if (logging_tablet_set->get_change_status()->is_change_logging()) {
       if (OB_SUCC(ret) && sync_result) {
         if (OB_FAIL(try_exec_special_op_(logging_tablet_set, scn, for_replay))) {
           DUP_TABLE_LOG(WARN, "try to execute special opertion for dup tablet set", K(ret),
                         KPC(logging_tablet_set));
+        } else if(logging_common_header.need_clean_all_readable_set())
+        {
+         clean_readable = true;
+        }
+
+        if (logging_common_header.is_old_set() && !logging_tablet_set->empty()) {
+          modify_readable_set = true;
         }
       }
       if (OB_SUCC(ret)) {
@@ -1251,7 +1270,7 @@ int ObLSDupTabletsMgr::tablet_log_synced(const bool sync_result,
         } else if (OB_FAIL(merge_into_readable_tablets_(logging_tablet_set, for_replay))) {
           DUP_TABLE_LOG(WARN, "merge into readable tablet set failed", K(ret));
         } else {
-          merge_confirmed = true;
+          modify_readable_set = true;
         }
       }
     }
@@ -1264,6 +1283,11 @@ int ObLSDupTabletsMgr::tablet_log_synced(const bool sync_result,
     }
   }
 
+  if (unique_id_array.count() > 0) {
+    DUP_TABLE_LOG(INFO, "tablet log sync", K(ret), K(sync_result), K(for_replay), K(is_master()),
+                  K(unique_id_array), K(scn), K(modify_readable_set));
+  }
+
   return ret;
 }
 
@@ -1334,7 +1358,8 @@ int ObLSDupTabletsMgr::merge_into_readable_tablets_(DupTabletChangeMap *change_m
 
   // an empty set first merge into readable list, return it to free pool
   if (OB_SUCC(ret)) {
-    if (OB_FAIL(check_and_recycle_empty_readable_set(change_map_ptr))) {
+    bool need_remove = false;
+    if (OB_FAIL(check_and_recycle_empty_readable_set(change_map_ptr, need_remove))) {
       DUP_TABLE_LOG(WARN, "return empty readable failed", K(ret), KPC(change_map_ptr));
     }
   }
@@ -2162,21 +2187,23 @@ int ObLSDupTabletsMgr::return_tablet_set(DupTabletChangeMap *need_free_set)
   return ret;
 }
 
-// remove emptry readable set
+// remove empty readable set
-int ObLSDupTabletsMgr::check_and_recycle_empty_readable_set(DupTabletChangeMap *readable_set)
+int ObLSDupTabletsMgr::check_and_recycle_empty_readable_set(DupTabletChangeMap *readable_set,
+                                                            bool &need_remove)
 {
   int ret = OB_SUCCESS;
 
+  need_remove = false;
   if (OB_ISNULL(readable_set) || !readable_set->get_common_header().is_readable_set()) {
     ret = OB_ERR_UNEXPECTED;
     DUP_TABLE_LOG(WARN, "unexpected tablet set", K(ret), KPC(readable_set));
   } else if (readable_set->empty()) {
-    DUP_TABLE_LOG(INFO, "try to remove empty readable tablet set from list", K(ret),
+    need_remove = true;
+    DUP_TABLE_LOG(INFO, "try to remove empty readable tablet set from list", K(ret), K(need_remove),
                   KPC(readable_set));
     if (OB_ISNULL(readable_tablets_list_.remove(readable_set))) {
       ret = OB_ERR_UNEXPECTED;
-      DUP_TABLE_LOG(WARN, "remove empty readable set from list failed", K(ret),
+      DUP_TABLE_LOG(WARN, "remove empty readable set from list failed", K(ret), KPC(readable_set));
-                    KPC(readable_set));
     } else if (OB_FAIL(return_tablet_set(readable_set))) {
       DUP_TABLE_LOG(WARN, "return empty readable set failed", K(ret), KPC(readable_set));
     }
@@ -2194,7 +2221,7 @@ int ObLSDupTabletsMgr::clean_readable_tablets_(const share::SCN &min_reserve_tab
     DUP_TABLE_LOG(WARN, "invalid argument", K(ret), K(ls_id_), K(min_reserve_tablet_scn));
   } else {
 
-    DUP_TABLE_LOG(INFO, "try to clean durable readable tablets", K(ret), K(ls_id_),
+    DUP_TABLE_LOG(INFO, "try to clean all readable tablets", K(ret), K(ls_id_),
                   K(min_reserve_tablet_scn), K(readable_tablets_list_.get_size()));
 
     DLIST_FOREACH_REMOVESAFE(readable_set, readable_tablets_list_)
@@ -2230,7 +2257,7 @@ int ObLSDupTabletsMgr::clean_durable_confirming_tablets_(const share::SCN &min_r
 
     DLIST_FOREACH_REMOVESAFE(need_confirm_set, need_confirm_new_queue_)
     {
-      DUP_TABLE_LOG(INFO, "try to clean one durable tablet set", K(ret), K(min_reserve_tablet_scn),
+      DUP_TABLE_LOG(INFO, "try to clean one confirming tablet set", K(ret), K(min_reserve_tablet_scn),
                     KPC(need_confirm_set));
       if (!need_confirm_set->need_reserve(min_reserve_tablet_scn)) {
         if (OB_ISNULL(need_confirm_new_queue_.remove(need_confirm_set))) {

src/storage/tx/ob_dup_table_tablets.h

@@ -628,7 +628,7 @@ public:
                         const share::SCN &scn,
                         const bool for_replay,
                         const DupTabletSetIDArray &unique_id_array,
-                        bool &merge_confirmed);
+                        bool &modify_readable_set);
 
   int try_to_confirm_tablets(const share::SCN &confirm_scn);
   // bool need_log_tablets();
@@ -764,7 +764,7 @@ private:
                              const bool construct_target_set = false,
                              const bool need_changing_new_set = false);
 
-  int check_and_recycle_empty_readable_set(DupTabletChangeMap *need_free_set);
+  int check_and_recycle_empty_readable_set(DupTabletChangeMap *need_free_set, bool &need_remove);
   int return_tablet_set(DupTabletChangeMap *need_free_set);
 
   int clean_readable_tablets_(const share::SCN & min_reserve_tablet_scn);
@@ -775,8 +775,9 @@ private:
   int clear_all_special_op_();
   int construct_clean_confirming_set_task_();
   int construct_clean_all_readable_set_task_();
-  int try_exec_special_op_(DupTabletChangeMap *op_tablet_set, const share::SCN &min_reserve_tablet_scn,const bool for_replay);
+  int try_exec_special_op_(DupTabletChangeMap *op_tablet_set,
-
+                           const share::SCN &min_reserve_tablet_scn,
+                           const bool for_replay);
   bool need_seralize_readable_set() { return true; }
 
   int cal_single_set_max_ser_size_(DupTabletChangeMap *hash_map,

src/storage/tx/ob_dup_table_util.cpp

@@ -1005,7 +1005,6 @@ int ObDupTableLSHandler::replay(const void *buffer,
     DUP_TABLE_LOG(WARN, "refresh dup table ls failed", K(tmp_ret), K(ls_id_), K(lsn), K(ts_ns));
   }
 
-  DUP_TABLE_LOG(DEBUG, "finish replay log", K(ret), K(ls_id_), K(lsn), K(ts_ns)); // for debug
   return ret;
 }