diff --git a/src/share/ob_ddl_common.cpp b/src/share/ob_ddl_common.cpp index 8ca3bffdb..5a9f1cbf2 100644 --- a/src/share/ob_ddl_common.cpp +++ b/src/share/ob_ddl_common.cpp @@ -1762,6 +1762,19 @@ int ObDDLUtil::check_schema_version_refreshed( return ret; } +bool ObDDLUtil::reach_time_interval(const int64_t i, volatile int64_t &last_time) +{ + bool bret = false; + const int64_t old_time = last_time; + const int64_t cur_time = common::ObTimeUtility::fast_current_time(); + if (OB_UNLIKELY((i + last_time) < cur_time) + && old_time == ATOMIC_CAS(&last_time, old_time, cur_time)) + { + bret = true; + } + return bret; +} + /****************** ObCheckTabletDataComplementOp *************/ int ObCheckTabletDataComplementOp::check_task_inner_sql_session_status( diff --git a/src/share/ob_ddl_common.h b/src/share/ob_ddl_common.h index 70bcec544..6030a0752 100644 --- a/src/share/ob_ddl_common.h +++ b/src/share/ob_ddl_common.h @@ -576,6 +576,9 @@ public: static int check_schema_version_refreshed( const uint64_t tenant_id, const int64_t target_schema_version); + + static bool reach_time_interval(const int64_t i, volatile int64_t &last_time); + private: static int generate_order_by_str( const ObIArray &select_column_ids, diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index a75d5ec91..3584d4bbd 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -45,7 +45,7 @@ int64_t ObTenantDirectLoadMgr::generate_context_id() } ObTenantDirectLoadMgr::ObTenantDirectLoadMgr() - : is_inited_(false), slice_id_generator_(0), context_id_generator_(0) + : is_inited_(false), slice_id_generator_(0), context_id_generator_(0), last_gc_time_(0) { } @@ -960,18 +960,18 @@ int ObTenantDirectLoadMgr::GetGcCandidateOp::operator() (common::hash::HashMapPa const ObTabletDirectLoadMgrKey &key = kv.first; ObTabletDirectLoadMgr *tablet_direct_load_mgr = kv.second; if (1 == tablet_direct_load_mgr->get_ref()) { - if (OB_FAIL(candidate_mgrs_.push_back(key))) { + if (OB_FAIL(candidate_mgrs_.push_back(std::make_pair(tablet_direct_load_mgr->get_ls_id(), key)))) { LOG_WARN("failed to push back", K(ret)); } } return ret; } -int ObTenantDirectLoadMgr::gc_tablet_direct_load(ObLS &ls) +int ObTenantDirectLoadMgr::gc_tablet_direct_load() { int ret = OB_SUCCESS; - if (!tablet_mgr_map_.empty()) { - ObSEArray candidate_mgrs; + if (!tablet_mgr_map_.empty() && ObDDLUtil::reach_time_interval(10 * 1000 * 1000, last_gc_time_)) { + ObSEArray, 8> candidate_mgrs; { ObBucketTryRLockAllGuard guard(bucket_lock_); if (OB_SUCC(guard.get_ret())) { @@ -980,12 +980,23 @@ int ObTenantDirectLoadMgr::gc_tablet_direct_load(ObLS &ls) } } - for (int64_t i = 0; i < candidate_mgrs.count(); i++) { - int tmp_ret = OB_SUCCESS; - ObTabletDirectLoadMgrKey mgr_key = candidate_mgrs.at(i); + for (int64_t i = 0; i < candidate_mgrs.count(); i++) { // overwrite ret + const share::ObLSID &ls_id = candidate_mgrs.at(i).first; + const ObTabletDirectLoadMgrKey &mgr_key = candidate_mgrs.at(i).second; + ObLSService *ls_svr = MTL(ObLSService*); + ObLS *ls = nullptr; + ObLSHandle ls_handle; ObTabletHandle tablet_handle; - if (OB_TMP_FAIL(ls.get_tablet(mgr_key.tablet_id_, tablet_handle))) { - LOG_WARN("failed to get tablet", K(ret), K(mgr_key)); + if (OB_ISNULL(ls_svr)) { + LOG_WARN("invalid mtl ObLSService", K(ret)); + } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::DDL_MOD))) { + LOG_WARN("get log stream failed", K(ret), K(ls_id)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("log stream not exist", K(ret)); + } else if (OB_FAIL(ls->get_tablet(mgr_key.tablet_id_, tablet_handle, + ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(mgr_key)); } else if (tablet_handle.get_obj()->get_major_table_count() > 0) { (void)remove_tablet_direct_load(mgr_key); } @@ -2497,7 +2508,7 @@ int ObTabletFullDirectLoadMgr::commit( } } if (OB_SUCC(ret) && lob_mgr_handle_.is_valid()) { - const ObLSID &ls_id = lob_mgr_handle_.get_full_obj()->get_ls_id(); + const share::ObLSID &ls_id = lob_mgr_handle_.get_full_obj()->get_ls_id(); const ObTabletID &lob_tablet_id = lob_mgr_handle_.get_full_obj()->get_tablet_id(); ObLSHandle ls_handle; ObLS *ls = nullptr; diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h index d10dfc9b4..957a0e7f5 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.h @@ -189,7 +189,7 @@ public: const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const bool is_full_direct_load); - int gc_tablet_direct_load(ObLS &ls); + int gc_tablet_direct_load(); // remove tablet direct load mgr from hashmap, // for full direct load, it will be called when physical major generates, // for incremental direct load, it will be called when all KVs dump. @@ -198,12 +198,13 @@ public: private: struct GetGcCandidateOp final { public: - GetGcCandidateOp(ObIArray &candidate_mgrs) : candidate_mgrs_(candidate_mgrs) {} + GetGcCandidateOp(ObIArray> &candidate_mgrs) + : candidate_mgrs_(candidate_mgrs) {} ~GetGcCandidateOp() {} int operator() (common::hash::HashMapPair &kv); private: DISALLOW_COPY_AND_ASSIGN(GetGcCandidateOp); - ObIArray &candidate_mgrs_; + ObIArray> &candidate_mgrs_; }; int try_create_tablet_direct_load_mgr( @@ -246,6 +247,7 @@ private: TABLET_EXEC_CONTEXT_MAP tablet_exec_context_map_; int64_t slice_id_generator_; int64_t context_id_generator_; + volatile int64_t last_gc_time_; DISALLOW_COPY_AND_ASSIGN(ObTenantDirectLoadMgr); }; diff --git a/src/storage/ls/ob_ls_ddl_log_handler.cpp b/src/storage/ls/ob_ls_ddl_log_handler.cpp index 8bfa34b6d..975dd9855 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.cpp +++ b/src/storage/ls/ob_ls_ddl_log_handler.cpp @@ -418,7 +418,6 @@ int ObLSDDLLogHandler::flush(SCN &rec_scn) } } } - (void)tenant_direct_load_mgr->gc_tablet_direct_load(*ls_); } } return OB_SUCCESS; @@ -460,6 +459,13 @@ SCN ObLSDDLLogHandler::get_rec_scn() } else if (!rec_scn.is_max()) { last_rec_scn_ = SCN::max(last_rec_scn_, rec_scn); } + + // gc tablet direct load periodically + ObTenantDirectLoadMgr *tenant_direct_load_mgr = MTL(ObTenantDirectLoadMgr *); + if (OB_NOT_NULL(tenant_direct_load_mgr)) { + (void)tenant_direct_load_mgr->gc_tablet_direct_load(); + } + LOG_INFO("[CHECKPOINT] ObLSDDLLogHandler::get_rec_scn", K(ret), "ls_id", OB_ISNULL(ls_) ? ObLSID() : ls_->get_ls_id(), K(barrier_tablet_id), K(rec_scn), K_(last_rec_scn));