From 0285badc2e83f215425fc52d5946bc0161639cbd Mon Sep 17 00:00:00 2001 From: "shenyunlong.syl" Date: Thu, 14 Sep 2023 12:13:50 +0000 Subject: [PATCH] [OBKV] Fix ttl issue 52136842 & 52111365 & 52218071 --- src/observer/table/ttl/ob_table_ttl_task.cpp | 33 +++-- src/observer/table/ttl/ob_table_ttl_task.h | 8 +- .../table/ttl/ob_tenant_tablet_ttl_mgr.cpp | 116 ++++++++++-------- .../table/ttl/ob_tenant_tablet_ttl_mgr.h | 21 +++- src/storage/ls/ob_ls.cpp | 6 + 5 files changed, 111 insertions(+), 73 deletions(-) diff --git a/src/observer/table/ttl/ob_table_ttl_task.cpp b/src/observer/table/ttl/ob_table_ttl_task.cpp index c7c91f3f0..76f848f92 100644 --- a/src/observer/table/ttl/ob_table_ttl_task.cpp +++ b/src/observer/table/ttl/ob_table_ttl_task.cpp @@ -37,7 +37,7 @@ ObTableTTLDeleteTask::ObTableTTLDeleteTask(): ObITask(ObITaskType::TASK_TYPE_TTL_DELETE), is_inited_(false), param_(), - info_(NULL), + info_(), allocator_(ObMemAttr(MTL_ID(), "TTLDelTaskCtx")), rowkey_(), ttl_tablet_mgr_(NULL), @@ -75,7 +75,8 @@ int ObTableTTLDeleteTask::init(ObTenantTabletTTLMgr *ttl_tablet_mgr, LOG_WARN("fail to init credential", KR(ret)); } else { param_ = ttl_para; - info_ = &ttl_info; + info_ = ttl_info; + info_.err_code_ = OB_SUCCESS; ttl_tablet_mgr_ = ttl_tablet_mgr; is_inited_ = true; } @@ -119,6 +120,12 @@ int ObTableTTLDeleteTask::process() LOG_WARN("not init", K(ret)); } else { bool need_stop = false; + // pre-check to quick cancel/suspend current task in case of tenant ttl state changed + // since it maybe wait in dag queue for too long + if (OB_FAIL(ttl_tablet_mgr_->report_task_status(info_, param_, need_stop, false))) { + LOG_WARN("fail to report ttl task status", KR(ret)); + } + // explicit cover retcode while(!need_stop) { lib::ContextParam param; param.set_mem_attr(MTL_ID(), "TTLDeleteMemCtx", ObCtxIds::DEFAULT_CTX_ID) @@ -129,13 +136,15 @@ int ObTableTTLDeleteTask::process() LOG_WARN("fail to process one", KR(ret)); } } - if (OB_FAIL(ttl_tablet_mgr_->report_task_status(const_cast(*info_), param_, need_stop))) { + // explicit cover retcode + if (OB_FAIL(ttl_tablet_mgr_->report_task_status(info_, param_, need_stop))) { LOG_WARN("fail to report ttl task status", KR(ret)); } allocator_.reuse(); } } } + ttl_tablet_mgr_->dec_dag_ref(); return ret; } @@ -150,8 +159,8 @@ int ObTableTTLDeleteTask::process_one() ObTableApiSpec *scan_spec = nullptr; observer::ObReqTimeGuard req_timeinfo_guard; // 引用cache资源必须加ObReqTimeGuard ObTableApiCacheGuard cache_guard; - ObTableTTLOperation ttl_operation(info_->tenant_id_, - info_->table_id_, + ObTableTTLOperation ttl_operation(info_.tenant_id_, + info_.table_id_, param_, PER_TASK_DEL_ROWS, rowkey_); @@ -207,15 +216,15 @@ int ObTableTTLDeleteTask::process_one() ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret; } - info_->max_version_del_cnt_ += result.get_max_version_del_row(); - info_->ttl_del_cnt_ += result.get_ttl_del_row(); - info_->scan_cnt_ += result.get_scan_row(); - info_->err_code_ = ret; - info_->row_key_ = result.get_end_rowkey(); + info_.max_version_del_cnt_ += result.get_max_version_del_row(); + info_.ttl_del_cnt_ += result.get_ttl_del_row(); + info_.scan_cnt_ += result.get_scan_row(); + info_.err_code_ = ret; + info_.row_key_ = result.get_end_rowkey(); if (OB_SUCC(ret) && result.get_del_row() < PER_TASK_DEL_ROWS) { ret = OB_ITER_END; // finsh task - info_->err_code_ = ret; - LOG_DEBUG("finish delete", KR(ret), KPC_(info)); + info_.err_code_ = ret; + LOG_DEBUG("finish delete", KR(ret), K_(info)); } int64_t cost = ObTimeUtil::current_time() - start_time; LOG_DEBUG("finish process one", KR(ret), K(cost)); diff --git a/src/observer/table/ttl/ob_table_ttl_task.h b/src/observer/table/ttl/ob_table_ttl_task.h index 87066982e..6d123ac78 100644 --- a/src/observer/table/ttl/ob_table_ttl_task.h +++ b/src/observer/table/ttl/ob_table_ttl_task.h @@ -97,11 +97,7 @@ public: virtual int process() override; common::ObTabletID get_tablet_id() const { - common::ObTabletID tablet_id; - if (OB_NOT_NULL(info_)) { - tablet_id = info_->get_tablet_id(); - } - return tablet_id; + return info_.get_tablet_id(); } uint64_t get_table_id() const { return param_.table_id_; } int64_t get_timeout_ts() { return ONE_TASK_TIMEOUT + ObTimeUtility::current_time(); } @@ -118,7 +114,7 @@ private: private: bool is_inited_; table::ObTTLTaskParam param_; - table::ObTTLTaskInfo *info_; + table::ObTTLTaskInfo info_; common::ObArenaAllocator allocator_; common::ObRowkey rowkey_; table::ObTenantTabletTTLMgr *ttl_tablet_mgr_; diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp index efa448ff4..163664946 100644 --- a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp @@ -73,14 +73,14 @@ int ObTenantTabletTTLMgr::init(const uint64_t tenant_id) tenant_id_ = tenant_id; is_inited_ = true; } - FLOG_INFO("finish init tenant_tablet_ttl_mgr", K_(tenant_id), K(ret)); + FLOG_INFO("finish init tenant_tablet_ttl_mgr", K_(tenant_id), K(ret), KPC_(ls)); return ret; } int ObTenantTabletTTLMgr::switch_to_leader() { int64_t start_time_us = ObTimeUtility::current_time(); - FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_leader", K_(tenant_id), K(start_time_us)); + FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_leader", K_(tenant_id), KPC_(ls), K(start_time_us)); int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat())) { LOG_WARN("fail to check_inner_stat", KR(ret), K_(tenant_id)); @@ -94,7 +94,7 @@ int ObTenantTabletTTLMgr::switch_to_leader() resume(); } const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; - FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_leader", KR(ret), K_(tenant_id), K(cost_us)); + FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_leader", KR(ret), K_(tenant_id), KPC_(ls), K(cost_us)); return ret; } @@ -123,11 +123,11 @@ void ObTenantTabletTTLMgr::switch_to_follower_forcedly() void ObTenantTabletTTLMgr::inner_switch_to_follower() { - FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_follower", K_(tenant_id)); + FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_follower", K_(tenant_id), KPC_(ls)); const int64_t start_time_us = ObTimeUtility::current_time(); pause(); const int64_t cost_us = ObTimeUtility::current_time() - start_time_us; - FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_follower", K_(tenant_id), K(cost_us)); + FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_follower", K_(tenant_id), KPC_(ls), K(cost_us)); } void ObTenantTabletTTLMgr::resume() @@ -143,7 +143,7 @@ void ObTenantTabletTTLMgr::pause() int ObTenantTabletTTLMgr::start() { int ret = OB_SUCCESS; - FLOG_INFO("tenant_tablet_ttl_mgr: begin to start", K_(tenant_id)); + FLOG_INFO("tenant_tablet_ttl_mgr: begin to start", KPC_(ls), K_(tenant_id)); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("tablet ttl mgr not init", KR(ret)); @@ -154,49 +154,37 @@ int ObTenantTabletTTLMgr::start() } else { is_timer_start_ = true; } - FLOG_INFO("tenant_tablet_ttl_mgr: finish to start", K(ret), K_(tenant_id)); + FLOG_INFO("tenant_tablet_ttl_mgr: finish to start", K(ret), KPC_(ls), K_(tenant_id)); return ret; } void ObTenantTabletTTLMgr::stop() { int ret = OB_SUCCESS; - FLOG_INFO("tenant_tablet_ttl_mgr: begin to stop", K_(tenant_id)); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("tablet ttl mgr not init", KR(ret)); - } else if (is_timer_start_) { + FLOG_INFO("tenant_tablet_ttl_mgr: begin to stop", K_(tenant_id), KPC_(ls)); + common::ObSpinLockGuard guard(lock_); + if (is_timer_start_) { TG_STOP(tg_id_); is_timer_start_ = false; + is_paused_ = true; } - FLOG_INFO("tenant_tablet_ttl_mgr: finish to stop", K(ret), K_(tenant_id)); + FLOG_INFO("tenant_tablet_ttl_mgr: finish to stop", K(ret), K_(is_timer_start), K_(tenant_id), KPC_(ls)); } void ObTenantTabletTTLMgr::wait() { - FLOG_INFO("tenant_tablet_ttl_mgr: begin to wait", K_(tenant_id)); - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("tablet ttl mgr not init", KR(ret)); - } else { - TG_WAIT(tg_id_); - } - FLOG_INFO("tenant_tablet_ttl_mgr: finish to wait", K_(tenant_id), K(ret)); + FLOG_INFO("tenant_tablet_ttl_mgr: begin to wait", KPC_(ls), K_(tenant_id)); + TG_WAIT(tg_id_); + FLOG_INFO("tenant_tablet_ttl_mgr: finish to wait", KPC_(ls), K_(tenant_id)); } void ObTenantTabletTTLMgr::destroy() { - int ret = OB_SUCCESS; - FLOG_INFO("tenant_tablet_ttl_mgr: begin to destroy", K_(tenant_id)); - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("tablet ttl mgr not init", KR(ret)); - } else { - TG_DESTROY(tg_id_); - is_inited_ = false; - } - FLOG_INFO("tenant_tablet_ttl_mgr: finish to destroy", K_(tenant_id), K(ret)); + FLOG_INFO("tenant_tablet_ttl_mgr: begin to destroy", KPC_(ls), K_(tenant_id)); + wait(); + TG_DESTROY(tg_id_); + is_inited_ = false; + FLOG_INFO("tenant_tablet_ttl_mgr: finish to destroy", KPC_(ls), K_(tenant_id)); } int ObTenantTabletTTLMgr::check_and_handle_event() @@ -205,6 +193,7 @@ int ObTenantTabletTTLMgr::check_and_handle_event() bool need_check = false; bool is_dirty = false; bool is_finished = false; + ObTimeGuard guard("ObTenantTabletTTLMgr::check_and_handle_event", TTL_NORMAL_TIME_THRESHOLD); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -231,13 +220,9 @@ int ObTenantTabletTTLMgr::check_and_handle_event() if (OB_SUCC(ret) && (is_dirty || !is_finished)) { common::ObSArray dirty_tasks; if (OB_FAIL(handle_all_tablet_event(dirty_tasks))) { - LOG_WARN("fail to handle tenant event in timer", K_(tenant_id)); - } - - for (int i = 0; OB_SUCC(ret) && i < dirty_tasks.count(); i++) { - if (OB_FAIL(sync_sys_table(dirty_tasks.at(i)))) { - LOG_WARN("fail to sync sys table", KR(ret)); - } + LOG_WARN("fail to handle tenant event in timer", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(sync_all_dirty_task(dirty_tasks))) { + LOG_WARN("fail to sync all dirty task", KR(ret), K_(tenant_id)); } check_ttl_tenant_state(); // check whether tenant still dirty or not @@ -361,7 +346,8 @@ void ObTenantTabletTTLMgr::on_schema_changed(uint64_t schema_changed_tenant_id) } } -int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTaskParam& task_para, bool& is_stop) +int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTaskParam& task_para, + bool& is_stop, bool need_copy_task/* true*/) { int ret = OB_SUCCESS; ObTTLTaskCtx* ctx = nullptr; @@ -385,8 +371,10 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask common::ObSpinLockGuard ctx_guard(ctx->lock_); ctx->last_modify_time_ = ObTimeUtility::current_time(); mark_ttl_ctx_dirty(local_tenant_task_, *ctx); - if (OB_FAIL(deep_copy_task(ctx, task_info, task_para))) { - LOG_WARN("fail to refresh ttl ctx content", KR(ret), K_(tenant_id)); + if (need_copy_task && OB_FAIL(deep_copy_task(ctx, task_info, task_para))) { + LOG_WARN("fail to deep copy task", KR(ret), K_(tenant_id), K(task_info), K(task_para)); + ctx->task_status_ = OB_TTL_TASK_PENDING; + ctx->failure_times_++; } else if (OB_SUCCESS == task_info.err_code_) { if (local_tenant_task_.ttl_continue_ && OB_TTL_TASK_RUNNING == local_tenant_task_.state_) { task_para = ctx->ttl_para_; @@ -401,7 +389,8 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask } else if (OB_NOT_MASTER == task_info.err_code_ || OB_PARTITION_NOT_EXIST == task_info.err_code_ || OB_TABLE_NOT_EXIST == task_info.err_code_ || - OB_ERR_UNKNOWN_TABLE == task_info.err_code_) { + OB_ERR_UNKNOWN_TABLE == task_info.err_code_ || + OB_LS_NOT_EXIST == task_info.err_code_) { LOG_INFO("cancel current task since partition state change", K(task_info.err_code_), K(task_info.tablet_id_)); ctx->task_status_ = OB_TTL_TASK_CANCEL; @@ -441,7 +430,7 @@ int ObTenantTabletTTLMgr::generate_batch_tablet_task(ObIArray table_schema_arr; @@ -581,7 +571,7 @@ int ObTenantTabletTTLMgr::check_and_generate_tablet_tasks() if (OB_SUCC(ret)) { mark_tenant_checked(); } - LOG_DEBUG("finish generate tenant tasks", KR(ret), K_(tenant_id)); + FLOG_INFO("finish generate tenant tasks", KR(ret), KPC_(ls), K_(tenant_id)); return ret; } @@ -589,6 +579,7 @@ void OBTTLTimerPeriodicTask::runTimerTask() { int ret = OB_SUCCESS; ObCurTraceId::init(GCONF.self_addr_); + ObTimeGuard guard("OBTTLTimerPeriodicTask::runTimerTask", TTL_TIME_TASKER_THRESHOLD); if (!ObKVFeatureModeUitl::is_ttl_enable()) { // do nothing LOG_DEBUG("ttl is disable"); @@ -624,7 +615,7 @@ int ObTenantTabletTTLMgr::alloc_tenant_info(uint64_t tenant_id) LOG_WARN("fail to create ttl partition map", KR(ret), K(tenant_id)); } else {} - FLOG_INFO("finish create tenant ttl task", K(tenant_id), K(ret)); + FLOG_INFO("finish create tenant ttl task", K(tenant_id), K(ret), KPC_(ls)); return ret; } @@ -708,6 +699,7 @@ int ObTenantTabletTTLMgr::generate_ttl_dag(ObTTLTaskInfo& task_info, ObTTLTaskPa int ObTenantTabletTTLMgr::handle_all_tablet_event(common::ObSArray& tablets) { int ret = OB_SUCCESS; + ObTimeGuard guard("ObTenantTabletTTLMgr::handle_all_tablet_event", TTL_NORMAL_TIME_THRESHOLD); ObTTLTaskCtx* ctx = nullptr; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -728,7 +720,7 @@ int ObTenantTabletTTLMgr::handle_all_tablet_event(common::ObSArray& } } - LOG_DEBUG("finish handle single tenant event", KR(ret), K_(tenant_id)); + FLOG_INFO("finish handle single tenant event", KR(ret), K_(tenant_id), KPC_(ls)); return ret; } @@ -889,6 +881,18 @@ int ObTenantTabletTTLMgr::try_schedule_prepare_task(ObTabletID& tablet_id) return ret; } +int ObTenantTabletTTLMgr::sync_all_dirty_task(ObIArray& dirty_tasks) +{ + int ret = OB_SUCCESS; + ObTimeGuard guard("ObTenantTabletTTLMgr::sync_all_dirty_record", TTL_NORMAL_TIME_THRESHOLD); + for (int i = 0; OB_SUCC(ret) && i < dirty_tasks.count(); i++) { + if (OB_FAIL(sync_sys_table(dirty_tasks.at(i)))) { + LOG_WARN("fail to sync sys table", KR(ret)); + } + } + return ret; +} + int ObTenantTabletTTLMgr::sync_sys_table(ObTabletID& tablet_id) { int ret = OB_SUCCESS; @@ -1116,7 +1120,6 @@ int ObTenantTabletTTLMgr::construct_task_record_filter(const uint64_t& task_id, int ObTenantTabletTTLMgr::from_ttl_record(ObTabletID& tablet_id, common::ObTTLStatus& record, bool with_status /*true*/, bool with_err_code /*true*/) { int ret = OB_SUCCESS; - common::ObSpinLockGuard guard(lock_); ObTTLTaskCtx* ctx = get_one_tablet_ctx(tablet_id); if (OB_ISNULL(ctx)) { ret = OB_ERR_NULL_VALUE; @@ -1168,7 +1171,7 @@ bool ObTenantTabletTTLMgr::can_schedule_tenant(const ObTTLTenantInfo &tenant_inf bool ObTenantTabletTTLMgr::can_schedule_task(const ObTTLTaskCtx &ttl_task) { - return ttl_task.task_status_ == OB_TTL_TASK_PENDING; + return !is_paused_ && ttl_task.task_status_ == OB_TTL_TASK_PENDING; } int ObTenantTabletTTLMgr::try_schedule_remaining_tasks(const ObTTLTaskCtx *current_ctx) @@ -1220,6 +1223,7 @@ int ObTenantTabletTTLMgr::try_schedule_task(ObTTLTaskCtx* ctx) LOG_WARN("fail to generate dag task", KR(ret)); } } else { + inc_dag_ref(); if (ctx->task_start_time_ == OB_INVALID_ID) { ctx->task_start_time_ = ObTimeUtility::current_time(); } @@ -1358,5 +1362,19 @@ int ObTenantTabletTTLMgr::check_schema_version() return ret; } +int ObTenantTabletTTLMgr::safe_to_destroy(bool &is_safe) +{ + int ret = OB_SUCCESS; + is_safe = true; + int64_t dag_ref = get_dag_ref(); + if (0 != dag_ref) { + if (REACH_TIME_INTERVAL(60L * 1000000)) { // 60s + LOG_WARN("tenant ttl tablet mgr can't destroy", K(dag_ref)); + } + is_safe = false; + } + return ret; +} + } // table } // oceanbase diff --git a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h index b7a4ed82a..f8862e4f8 100644 --- a/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h +++ b/src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h @@ -67,6 +67,7 @@ public: virtual ~OBTTLTimerPeriodicTask() {} virtual void runTimerTask() override; private: + static const int64_t TTL_TIME_TASKER_THRESHOLD = 30 * 1000 * 1000; // 30s ObTenantTabletTTLMgr &tablet_ttl_mgr_; }; @@ -77,10 +78,10 @@ class ObTenantTabletTTLMgr : public logservice::ObIReplaySubHandler, public: friend ObTTLTaskCtx; ObTenantTabletTTLMgr() - : tenant_id_(common::OB_INVALID_TENANT_ID), + : allocator_(ObMemAttr(MTL_ID(), "TenantTTLMgr")), + tenant_id_(common::OB_INVALID_TENANT_ID), schema_service_(NULL), sql_proxy_(NULL), - allocator_(ObMemAttr(MTL_ID(), "TenantTTLMgr")), is_inited_(false), is_timer_start_(false), periodic_delay_(TTL_PERIODIC_DELAY), @@ -89,7 +90,8 @@ public: tg_id_(0), local_schema_version_(OB_INVALID_VERSION), has_start_(false), - is_paused_(false) + is_paused_(false), + dag_ref_cnt_(0) { } @@ -137,7 +139,8 @@ public: int reload_tenant_task(); int report_task_status(ObTTLTaskInfo& task_info, table::ObTTLTaskParam& task_para, - bool& is_stop); + bool& is_stop, + bool need_copy_task = true); void on_schema_changed(uint64_t schema_changed_tenant_id); // timer handle function @@ -145,6 +148,11 @@ public: int check_and_handle_event(); int check_tenant_memory(); int check_inner_stat(); + void inc_dag_ref() { ATOMIC_INC(&dag_ref_cnt_); } + void dec_dag_ref() { ATOMIC_DEC(&dag_ref_cnt_); } + int64_t get_dag_ref() const { return ATOMIC_LOAD(&dag_ref_cnt_); } + int safe_to_destroy(bool &is_safe); + int sync_all_dirty_task(common::ObIArray& dirty_tasks); private: typedef common::hash::ObHashMap TabletTaskMap; typedef TabletTaskMap::iterator tablet_task_iter; @@ -249,7 +257,6 @@ private: int check_schema_version(); void resume(); void pause(); - private: static const int64_t DEFAULT_TTL_BUCKET_NUM = 100; static const int64_t TTL_PERIODIC_DELAY = 5*1000*1000; //5s @@ -257,11 +264,12 @@ private: static const int64_t DEFAULT_TABLE_ARRAY_SIZE = 200; static const int64_t DEFAULT_TABLET_PAIR_SIZE = 1024; static const int64_t DEFAULT_PARAM_BUCKET_SIZE = 200; + static const int64_t TTL_NORMAL_TIME_THRESHOLD = 3*1000*1000; // 3s + common::ObArenaAllocator allocator_; uint64_t tenant_id_; ObTTLTenantInfo local_tenant_task_; share::schema::ObMultiVersionSchemaService *schema_service_; common::ObMySQLProxy *sql_proxy_; - common::ObArenaAllocator allocator_; bool is_inited_; bool is_timer_start_; int64_t periodic_delay_; @@ -273,6 +281,7 @@ private: int64_t local_schema_version_; bool has_start_; bool is_paused_; + volatile int64_t dag_ref_cnt_; // ttl dag ref count for current ls }; } // end namespace table diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 78e703d45..450f53eb7 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -597,6 +597,7 @@ int ObLS::stop_() } ls_migration_handler_.stop(); ls_tablet_svr_.stop(); + tablet_ttl_mgr_.stop(); is_stopped_ = true; if (OB_SUCC(ret)) { @@ -687,6 +688,7 @@ bool ObLS::safe_to_destroy() bool is_dup_table_handler_safe = false; bool is_log_handler_safe = false; bool is_transfer_handler_safe = false; + bool is_ttl_mgr_safe = false; if (OB_FAIL(ls_tablet_svr_.safe_to_destroy(is_tablet_service_safe))) { LOG_WARN("ls tablet service check safe to destroy failed", K(ret), KPC(this)); @@ -706,6 +708,9 @@ bool ObLS::safe_to_destroy() } else if (OB_FAIL(transfer_handler_.safe_to_destroy(is_transfer_handler_safe))) { LOG_WARN("transfer_handler_ check safe to destroy failed", K(ret), KPC(this)); } else if (!is_transfer_handler_safe) { + } else if (OB_FAIL(tablet_ttl_mgr_.safe_to_destroy(is_ttl_mgr_safe))) { + LOG_WARN("tablet_ttl_mgr_ check safe to destroy failed", K(ret), KPC(this)); + } else if (!is_ttl_mgr_safe) { } else { if (1 == ref_mgr_.get_total_ref_cnt()) { // only has one ref at the safe destroy task is_safe = true; @@ -721,6 +726,7 @@ bool ObLS::safe_to_destroy() K(is_dup_table_handler_safe), K(is_ls_restore_handler_safe), K(is_log_handler_safe), K(is_transfer_handler_safe), + K(is_ttl_mgr_safe), "ls_ref", ref_mgr_.get_total_ref_cnt(), K(ret), KP(this), KPC(this)); ref_mgr_.print();