gc_tablet_direct_load in get_rec_scn

src/share/ob_ddl_common.cpp

@@ -1762,6 +1762,19 @@ int ObDDLUtil::check_schema_version_refreshed(
   return ret;
 }
 
+bool ObDDLUtil::reach_time_interval(const int64_t i, volatile int64_t &last_time)
+{
+  bool bret = false;
+  const int64_t old_time = last_time;
+  const int64_t cur_time = common::ObTimeUtility::fast_current_time();
+  if (OB_UNLIKELY((i + last_time) < cur_time)
+      && old_time == ATOMIC_CAS(&last_time, old_time, cur_time))
+  {
+    bret = true;
+  }
+  return bret;
+}
+
 /******************           ObCheckTabletDataComplementOp         *************/
 
 int ObCheckTabletDataComplementOp::check_task_inner_sql_session_status(

src/share/ob_ddl_common.h

@@ -576,6 +576,9 @@ public:
   static int check_schema_version_refreshed(
       const uint64_t tenant_id,
       const int64_t target_schema_version);
+
+  static bool reach_time_interval(const int64_t i, volatile int64_t &last_time);
+
 private:
   static int generate_order_by_str(
       const ObIArray<int64_t> &select_column_ids,

src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp

@@ -45,7 +45,7 @@ int64_t ObTenantDirectLoadMgr::generate_context_id()
 }
 
 ObTenantDirectLoadMgr::ObTenantDirectLoadMgr()
-  : is_inited_(false), slice_id_generator_(0), context_id_generator_(0)
+  : is_inited_(false), slice_id_generator_(0), context_id_generator_(0), last_gc_time_(0)
 {
 }
 
@@ -960,18 +960,18 @@ int ObTenantDirectLoadMgr::GetGcCandidateOp::operator() (common::hash::HashMapPa
   const ObTabletDirectLoadMgrKey &key = kv.first;
   ObTabletDirectLoadMgr *tablet_direct_load_mgr = kv.second;
   if (1 == tablet_direct_load_mgr->get_ref()) {
-    if (OB_FAIL(candidate_mgrs_.push_back(key))) {
+    if (OB_FAIL(candidate_mgrs_.push_back(std::make_pair(tablet_direct_load_mgr->get_ls_id(), key)))) {
       LOG_WARN("failed to push back", K(ret));
     }
   }
   return ret;
 }
 
-int ObTenantDirectLoadMgr::gc_tablet_direct_load(ObLS &ls)
+int ObTenantDirectLoadMgr::gc_tablet_direct_load()
 {
   int ret = OB_SUCCESS;
-  if (!tablet_mgr_map_.empty()) {
+  if (!tablet_mgr_map_.empty() && ObDDLUtil::reach_time_interval(10 * 1000 * 1000, last_gc_time_)) {
-    ObSEArray<ObTabletDirectLoadMgrKey, 8> candidate_mgrs;
+    ObSEArray<std::pair<share::ObLSID, ObTabletDirectLoadMgrKey>, 8> candidate_mgrs;
     {
       ObBucketTryRLockAllGuard guard(bucket_lock_);
       if (OB_SUCC(guard.get_ret())) {
@@ -980,12 +980,23 @@ int ObTenantDirectLoadMgr::gc_tablet_direct_load(ObLS &ls)
       }
     }
 
-    for (int64_t i = 0; i < candidate_mgrs.count(); i++) {
+    for (int64_t i = 0; i < candidate_mgrs.count(); i++) { // overwrite ret
-      int tmp_ret = OB_SUCCESS;
+      const share::ObLSID &ls_id = candidate_mgrs.at(i).first;
-      ObTabletDirectLoadMgrKey mgr_key = candidate_mgrs.at(i);
+      const ObTabletDirectLoadMgrKey &mgr_key = candidate_mgrs.at(i).second;
+      ObLSService *ls_svr = MTL(ObLSService*);
+      ObLS *ls = nullptr;
+      ObLSHandle ls_handle;
       ObTabletHandle tablet_handle;
-      if (OB_TMP_FAIL(ls.get_tablet(mgr_key.tablet_id_, tablet_handle))) {
+      if (OB_ISNULL(ls_svr)) {
-        LOG_WARN("failed to get tablet", K(ret), K(mgr_key));
+        LOG_WARN("invalid mtl ObLSService", K(ret));
+      } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) {
+        LOG_WARN("get log stream failed", K(ret), K(ls_id));
+      } else if (OB_ISNULL(ls = ls_handle.get_ls())) {
+        ret = OB_ERR_UNEXPECTED;
+        LOG_WARN("log stream not exist", K(ret));
+      } else if (OB_FAIL(ls->get_tablet(mgr_key.tablet_id_, tablet_handle,
+              ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_ALL_COMMITED))) {
+        LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(mgr_key));
       } else if (tablet_handle.get_obj()->get_major_table_count() > 0) {
         (void)remove_tablet_direct_load(mgr_key);
       }
@@ -2497,7 +2508,7 @@ int ObTabletFullDirectLoadMgr::commit(
     }
   }
   if (OB_SUCC(ret) && lob_mgr_handle_.is_valid()) {
-    const ObLSID &ls_id = lob_mgr_handle_.get_full_obj()->get_ls_id();
+    const share::ObLSID &ls_id = lob_mgr_handle_.get_full_obj()->get_ls_id();
     const ObTabletID &lob_tablet_id = lob_mgr_handle_.get_full_obj()->get_tablet_id();
     ObLSHandle ls_handle;
     ObLS *ls = nullptr;

src/storage/ddl/ob_direct_insert_sstable_ctx_new.h

@@ -189,7 +189,7 @@ public:
       const share::ObLSID &ls_id,
       const common::ObTabletID &tablet_id,
       const bool is_full_direct_load);
-  int gc_tablet_direct_load(ObLS &ls);
+  int gc_tablet_direct_load();
   // remove tablet direct load mgr from hashmap,
   // for full direct load, it will be called when physical major generates,
   // for incremental direct load, it will be called when all KVs dump.
@@ -198,12 +198,13 @@ public:
 private:
   struct GetGcCandidateOp final {
   public:
-    GetGcCandidateOp(ObIArray<ObTabletDirectLoadMgrKey> &candidate_mgrs) : candidate_mgrs_(candidate_mgrs) {}
+    GetGcCandidateOp(ObIArray<std::pair<share::ObLSID, ObTabletDirectLoadMgrKey>> &candidate_mgrs)
+      : candidate_mgrs_(candidate_mgrs) {}
     ~GetGcCandidateOp() {}
     int operator() (common::hash::HashMapPair<ObTabletDirectLoadMgrKey, ObTabletDirectLoadMgr *> &kv);
   private:
     DISALLOW_COPY_AND_ASSIGN(GetGcCandidateOp);
-    ObIArray<ObTabletDirectLoadMgrKey> &candidate_mgrs_;
+    ObIArray<std::pair<share::ObLSID, ObTabletDirectLoadMgrKey>> &candidate_mgrs_;
   };
 
   int try_create_tablet_direct_load_mgr(
@@ -246,6 +247,7 @@ private:
   TABLET_EXEC_CONTEXT_MAP tablet_exec_context_map_;
   int64_t slice_id_generator_;
   int64_t context_id_generator_;
+  volatile int64_t last_gc_time_;
 DISALLOW_COPY_AND_ASSIGN(ObTenantDirectLoadMgr);
 };
 

src/storage/ls/ob_ls_ddl_log_handler.cpp

@@ -418,7 +418,6 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn)
           }
         }
       }
-      (void)tenant_direct_load_mgr->gc_tablet_direct_load(*ls_);
     }
   }
   return OB_SUCCESS;
@@ -460,6 +459,13 @@ SCN ObLSDDLLogHandler::get_rec_scn()
   } else if (!rec_scn.is_max()) {
     last_rec_scn_ = SCN::max(last_rec_scn_, rec_scn);
   }
+
+  // gc tablet direct load periodically
+  ObTenantDirectLoadMgr *tenant_direct_load_mgr = MTL(ObTenantDirectLoadMgr *);
+  if (OB_NOT_NULL(tenant_direct_load_mgr)) {
+    (void)tenant_direct_load_mgr->gc_tablet_direct_load();
+  }
+
   LOG_INFO("[CHECKPOINT] ObLSDDLLogHandler::get_rec_scn", K(ret),
       "ls_id", OB_ISNULL(ls_) ? ObLSID() : ls_->get_ls_id(),
       K(barrier_tablet_id), K(rec_scn), K_(last_rec_scn));