hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[OBKV] Fix ttl issue 52136842 & 52111365 & 52218071

Browse Source
This commit is contained in:
shenyunlong.syl 2023-09-14 12:13:50 +00:00 committed by ob-robot
parent 00b4c3b1c4
commit 0285badc2e
5 changed files with 111 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/observer/table/ttl/ob_table_ttl_task.cpp
View File

@ -37,7 +37,7 @@ ObTableTTLDeleteTask::ObTableTTLDeleteTask():
ObITask(ObITaskType::TASK_TYPE_TTL_DELETE),
is_inited_(false),
param_(),
info_(NULL),
info_(),
allocator_(ObMemAttr(MTL_ID(), "TTLDelTaskCtx")),
rowkey_(),
ttl_tablet_mgr_(NULL),
@ -75,7 +75,8 @@ int ObTableTTLDeleteTask::init(ObTenantTabletTTLMgr *ttl_tablet_mgr,
LOG_WARN("fail to init credential", KR(ret));
} else {
param_ = ttl_para;
info_ = &ttl_info;
info_ = ttl_info;
info_.err_code_ = OB_SUCCESS;
ttl_tablet_mgr_ = ttl_tablet_mgr;
is_inited_ = true;
}
@ -119,6 +120,12 @@ int ObTableTTLDeleteTask::process()
LOG_WARN("not init", K(ret));
} else {
bool need_stop = false;
// pre-check to quick cancel/suspend current task in case of tenant ttl state changed
// since it maybe wait in dag queue for too long
if (OB_FAIL(ttl_tablet_mgr_->report_task_status(info_, param_, need_stop, false))) {
LOG_WARN("fail to report ttl task status", KR(ret));
}
// explicit cover retcode
while(!need_stop) {
lib::ContextParam param;
param.set_mem_attr(MTL_ID(), "TTLDeleteMemCtx", ObCtxIds::DEFAULT_CTX_ID)
@ -129,13 +136,15 @@ int ObTableTTLDeleteTask::process()
LOG_WARN("fail to process one", KR(ret));
}
}
if (OB_FAIL(ttl_tablet_mgr_->report_task_status(const_cast<ObTTLTaskInfo&>(*info_), param_, need_stop))) {
// explicit cover retcode
if (OB_FAIL(ttl_tablet_mgr_->report_task_status(info_, param_, need_stop))) {
LOG_WARN("fail to report ttl task status", KR(ret));
}
allocator_.reuse();
}
}
}
ttl_tablet_mgr_->dec_dag_ref();
return ret;
}
@ -150,8 +159,8 @@ int ObTableTTLDeleteTask::process_one()
ObTableApiSpec *scan_spec = nullptr;
observer::ObReqTimeGuard req_timeinfo_guard; // 引用cache资源必须加ObReqTimeGuard
ObTableApiCacheGuard cache_guard;
ObTableTTLOperation ttl_operation(info_->tenant_id_,
info_->table_id_,
ObTableTTLOperation ttl_operation(info_.tenant_id_,
info_.table_id_,
param_,
PER_TASK_DEL_ROWS,
rowkey_);
@ -207,15 +216,15 @@ int ObTableTTLDeleteTask::process_one()
ret = (OB_SUCCESS == tmp_ret) ? ret : tmp_ret;
}
info_->max_version_del_cnt_ += result.get_max_version_del_row();
info_->ttl_del_cnt_ += result.get_ttl_del_row();
info_->scan_cnt_ += result.get_scan_row();
info_->err_code_ = ret;
info_->row_key_ = result.get_end_rowkey();
info_.max_version_del_cnt_ += result.get_max_version_del_row();
info_.ttl_del_cnt_ += result.get_ttl_del_row();
info_.scan_cnt_ += result.get_scan_row();
info_.err_code_ = ret;
info_.row_key_ = result.get_end_rowkey();
if (OB_SUCC(ret) && result.get_del_row() < PER_TASK_DEL_ROWS) {
ret = OB_ITER_END; // finsh task
info_->err_code_ = ret;
LOG_DEBUG("finish delete", KR(ret), KPC_(info));
info_.err_code_ = ret;
LOG_DEBUG("finish delete", KR(ret), K_(info));
}
int64_t cost = ObTimeUtil::current_time() - start_time;
LOG_DEBUG("finish process one", KR(ret), K(cost));

8
src/observer/table/ttl/ob_table_ttl_task.h
View File

@ -97,11 +97,7 @@ public:
virtual int process() override;
common::ObTabletID get_tablet_id() const
{
common::ObTabletID tablet_id;
if (OB_NOT_NULL(info_)) {
tablet_id = info_->get_tablet_id();
}
return tablet_id;
return info_.get_tablet_id();
}
uint64_t get_table_id() const { return param_.table_id_; }
int64_t get_timeout_ts() { return ONE_TASK_TIMEOUT + ObTimeUtility::current_time(); }
@ -118,7 +114,7 @@ private:
private:
bool is_inited_;
table::ObTTLTaskParam param_;
table::ObTTLTaskInfo *info_;
table::ObTTLTaskInfo info_;
common::ObArenaAllocator allocator_;
common::ObRowkey rowkey_;
table::ObTenantTabletTTLMgr *ttl_tablet_mgr_;

116
src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.cpp
View File

@ -73,14 +73,14 @@ int ObTenantTabletTTLMgr::init(const uint64_t tenant_id)
tenant_id_ = tenant_id;
is_inited_ = true;
}
FLOG_INFO("finish init tenant_tablet_ttl_mgr", K_(tenant_id), K(ret));
FLOG_INFO("finish init tenant_tablet_ttl_mgr", K_(tenant_id), K(ret), KPC_(ls));
return ret;
}
int ObTenantTabletTTLMgr::switch_to_leader()
{
int64_t start_time_us = ObTimeUtility::current_time();
FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_leader", K_(tenant_id), K(start_time_us));
FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_leader", K_(tenant_id), KPC_(ls), K(start_time_us));
int ret = OB_SUCCESS;
if (OB_FAIL(check_inner_stat())) {
LOG_WARN("fail to check_inner_stat", KR(ret), K_(tenant_id));
@ -94,7 +94,7 @@ int ObTenantTabletTTLMgr::switch_to_leader()
resume();
}
const int64_t cost_us = ObTimeUtility::current_time() - start_time_us;
FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_leader", KR(ret), K_(tenant_id), K(cost_us));
FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_leader", KR(ret), K_(tenant_id), KPC_(ls), K(cost_us));
return ret;
}
@ -123,11 +123,11 @@ void ObTenantTabletTTLMgr::switch_to_follower_forcedly()
void ObTenantTabletTTLMgr::inner_switch_to_follower()
{
FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_follower", K_(tenant_id));
FLOG_INFO("tenant_tablet_ttl_mgr: begin to switch_to_follower", K_(tenant_id), KPC_(ls));
const int64_t start_time_us = ObTimeUtility::current_time();
pause();
const int64_t cost_us = ObTimeUtility::current_time() - start_time_us;
FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_follower", K_(tenant_id), K(cost_us));
FLOG_INFO("tenant_tablet_ttl_mgr: finish to switch_to_follower", K_(tenant_id), KPC_(ls), K(cost_us));
}
void ObTenantTabletTTLMgr::resume()
@ -143,7 +143,7 @@ void ObTenantTabletTTLMgr::pause()
int ObTenantTabletTTLMgr::start()
{
int ret = OB_SUCCESS;
FLOG_INFO("tenant_tablet_ttl_mgr: begin to start", K_(tenant_id));
FLOG_INFO("tenant_tablet_ttl_mgr: begin to start", KPC_(ls), K_(tenant_id));
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("tablet ttl mgr not init", KR(ret));
@ -154,49 +154,37 @@ int ObTenantTabletTTLMgr::start()
} else {
is_timer_start_ = true;
}
FLOG_INFO("tenant_tablet_ttl_mgr: finish to start", K(ret), K_(tenant_id));
FLOG_INFO("tenant_tablet_ttl_mgr: finish to start", K(ret), KPC_(ls), K_(tenant_id));
return ret;
}
void ObTenantTabletTTLMgr::stop()
{
int ret = OB_SUCCESS;
FLOG_INFO("tenant_tablet_ttl_mgr: begin to stop", K_(tenant_id));
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("tablet ttl mgr not init", KR(ret));
} else if (is_timer_start_) {
FLOG_INFO("tenant_tablet_ttl_mgr: begin to stop", K_(tenant_id), KPC_(ls));
common::ObSpinLockGuard guard(lock_);
if (is_timer_start_) {
TG_STOP(tg_id_);
is_timer_start_ = false;
is_paused_ = true;
}
FLOG_INFO("tenant_tablet_ttl_mgr: finish to stop", K(ret), K_(tenant_id));
FLOG_INFO("tenant_tablet_ttl_mgr: finish to stop", K(ret), K_(is_timer_start), K_(tenant_id), KPC_(ls));
}
void ObTenantTabletTTLMgr::wait()
{
FLOG_INFO("tenant_tablet_ttl_mgr: begin to wait", K_(tenant_id));
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("tablet ttl mgr not init", KR(ret));
} else {
TG_WAIT(tg_id_);
}
FLOG_INFO("tenant_tablet_ttl_mgr: finish to wait", K_(tenant_id), K(ret));
FLOG_INFO("tenant_tablet_ttl_mgr: begin to wait", KPC_(ls), K_(tenant_id));
TG_WAIT(tg_id_);
FLOG_INFO("tenant_tablet_ttl_mgr: finish to wait", KPC_(ls), K_(tenant_id));
}
void ObTenantTabletTTLMgr::destroy()
{
int ret = OB_SUCCESS;
FLOG_INFO("tenant_tablet_ttl_mgr: begin to destroy", K_(tenant_id));
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("tablet ttl mgr not init", KR(ret));
} else {
TG_DESTROY(tg_id_);
is_inited_ = false;
}
FLOG_INFO("tenant_tablet_ttl_mgr: finish to destroy", K_(tenant_id), K(ret));
FLOG_INFO("tenant_tablet_ttl_mgr: begin to destroy", KPC_(ls), K_(tenant_id));
wait();
TG_DESTROY(tg_id_);
is_inited_ = false;
FLOG_INFO("tenant_tablet_ttl_mgr: finish to destroy", KPC_(ls), K_(tenant_id));
}
int ObTenantTabletTTLMgr::check_and_handle_event()
@ -205,6 +193,7 @@ int ObTenantTabletTTLMgr::check_and_handle_event()
bool need_check = false;
bool is_dirty = false;
bool is_finished = false;
ObTimeGuard guard("ObTenantTabletTTLMgr::check_and_handle_event", TTL_NORMAL_TIME_THRESHOLD);
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -231,13 +220,9 @@ int ObTenantTabletTTLMgr::check_and_handle_event()
if (OB_SUCC(ret) && (is_dirty || !is_finished)) {
common::ObSArray<ObTabletID> dirty_tasks;
if (OB_FAIL(handle_all_tablet_event(dirty_tasks))) {
LOG_WARN("fail to handle tenant event in timer", K_(tenant_id));
}
for (int i = 0; OB_SUCC(ret) && i < dirty_tasks.count(); i++) {
if (OB_FAIL(sync_sys_table(dirty_tasks.at(i)))) {
LOG_WARN("fail to sync sys table", KR(ret));
}
LOG_WARN("fail to handle tenant event in timer", KR(ret), K_(tenant_id));
} else if (OB_FAIL(sync_all_dirty_task(dirty_tasks))) {
LOG_WARN("fail to sync all dirty task", KR(ret), K_(tenant_id));
}
check_ttl_tenant_state(); // check whether tenant still dirty or not
@ -361,7 +346,8 @@ void ObTenantTabletTTLMgr::on_schema_changed(uint64_t schema_changed_tenant_id)
}
}
int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTaskParam& task_para, bool& is_stop)
int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTaskParam& task_para,
bool& is_stop, bool need_copy_task/* true*/)
{
int ret = OB_SUCCESS;
ObTTLTaskCtx* ctx = nullptr;
@ -385,8 +371,10 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask
common::ObSpinLockGuard ctx_guard(ctx->lock_);
ctx->last_modify_time_ = ObTimeUtility::current_time();
mark_ttl_ctx_dirty(local_tenant_task_, *ctx);
if (OB_FAIL(deep_copy_task(ctx, task_info, task_para))) {
LOG_WARN("fail to refresh ttl ctx content", KR(ret), K_(tenant_id));
if (need_copy_task && OB_FAIL(deep_copy_task(ctx, task_info, task_para))) {
LOG_WARN("fail to deep copy task", KR(ret), K_(tenant_id), K(task_info), K(task_para));
ctx->task_status_ = OB_TTL_TASK_PENDING;
ctx->failure_times_++;
} else if (OB_SUCCESS == task_info.err_code_) {
if (local_tenant_task_.ttl_continue_ && OB_TTL_TASK_RUNNING == local_tenant_task_.state_) {
task_para = ctx->ttl_para_;
@ -401,7 +389,8 @@ int ObTenantTabletTTLMgr::report_task_status(ObTTLTaskInfo& task_info, ObTTLTask
} else if (OB_NOT_MASTER == task_info.err_code_ ||
OB_PARTITION_NOT_EXIST == task_info.err_code_ ||
OB_TABLE_NOT_EXIST == task_info.err_code_ ||
OB_ERR_UNKNOWN_TABLE == task_info.err_code_) {
OB_ERR_UNKNOWN_TABLE == task_info.err_code_ ||
OB_LS_NOT_EXIST == task_info.err_code_) {
LOG_INFO("cancel current task since partition state change",
K(task_info.err_code_), K(task_info.tablet_id_));
ctx->task_status_ = OB_TTL_TASK_CANCEL;
@ -441,7 +430,7 @@ int ObTenantTabletTTLMgr::generate_batch_tablet_task(ObIArray<share::ObTabletTab
}
}
FLOG_INFO("finish generate tablet task", KR(ret), K(tablet_pairs));
FLOG_INFO("finish generate tablet task", KR(ret), KPC_(ls), K(tablet_pairs));
return ret;
}
@ -496,6 +485,7 @@ void ObTenantTabletTTLMgr::mark_tenant_checked()
int ObTenantTabletTTLMgr::check_and_generate_tablet_tasks()
{
int ret = OB_SUCCESS;
ObTimeGuard guard("ObTenantTabletTTLMgr::check_and_generate_tablet_tasks", TTL_NORMAL_TIME_THRESHOLD);
bool can_ttl = false;
ObTabletHandle tablet_handle;
ObSEArray<const schema::ObTableSchema *, 64> table_schema_arr;
@ -581,7 +571,7 @@ int ObTenantTabletTTLMgr::check_and_generate_tablet_tasks()
if (OB_SUCC(ret)) {
mark_tenant_checked();
}
LOG_DEBUG("finish generate tenant tasks", KR(ret), K_(tenant_id));
FLOG_INFO("finish generate tenant tasks", KR(ret), KPC_(ls), K_(tenant_id));
return ret;
}
@ -589,6 +579,7 @@ void OBTTLTimerPeriodicTask::runTimerTask()
{
int ret = OB_SUCCESS;
ObCurTraceId::init(GCONF.self_addr_);
ObTimeGuard guard("OBTTLTimerPeriodicTask::runTimerTask", TTL_TIME_TASKER_THRESHOLD);
if (!ObKVFeatureModeUitl::is_ttl_enable()) {
// do nothing
LOG_DEBUG("ttl is disable");
@ -624,7 +615,7 @@ int ObTenantTabletTTLMgr::alloc_tenant_info(uint64_t tenant_id)
LOG_WARN("fail to create ttl partition map", KR(ret), K(tenant_id));
} else {}
FLOG_INFO("finish create tenant ttl task", K(tenant_id), K(ret));
FLOG_INFO("finish create tenant ttl task", K(tenant_id), K(ret), KPC_(ls));
return ret;
}
@ -708,6 +699,7 @@ int ObTenantTabletTTLMgr::generate_ttl_dag(ObTTLTaskInfo& task_info, ObTTLTaskPa
int ObTenantTabletTTLMgr::handle_all_tablet_event(common::ObSArray<ObTabletID>& tablets)
{
int ret = OB_SUCCESS;
ObTimeGuard guard("ObTenantTabletTTLMgr::handle_all_tablet_event", TTL_NORMAL_TIME_THRESHOLD);
ObTTLTaskCtx* ctx = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
@ -728,7 +720,7 @@ int ObTenantTabletTTLMgr::handle_all_tablet_event(common::ObSArray<ObTabletID>&
}
}
LOG_DEBUG("finish handle single tenant event", KR(ret), K_(tenant_id));
FLOG_INFO("finish handle single tenant event", KR(ret), K_(tenant_id), KPC_(ls));
return ret;
}
@ -889,6 +881,18 @@ int ObTenantTabletTTLMgr::try_schedule_prepare_task(ObTabletID& tablet_id)
return ret;
}
int ObTenantTabletTTLMgr::sync_all_dirty_task(ObIArray<ObTabletID>& dirty_tasks)
{
int ret = OB_SUCCESS;
ObTimeGuard guard("ObTenantTabletTTLMgr::sync_all_dirty_record", TTL_NORMAL_TIME_THRESHOLD);
for (int i = 0; OB_SUCC(ret) && i < dirty_tasks.count(); i++) {
if (OB_FAIL(sync_sys_table(dirty_tasks.at(i)))) {
LOG_WARN("fail to sync sys table", KR(ret));
}
}
return ret;
}
int ObTenantTabletTTLMgr::sync_sys_table(ObTabletID& tablet_id)
{
int ret = OB_SUCCESS;
@ -1116,7 +1120,6 @@ int ObTenantTabletTTLMgr::construct_task_record_filter(const uint64_t& task_id,
int ObTenantTabletTTLMgr::from_ttl_record(ObTabletID& tablet_id, common::ObTTLStatus& record, bool with_status /*true*/, bool with_err_code /*true*/)
{
int ret = OB_SUCCESS;
common::ObSpinLockGuard guard(lock_);
ObTTLTaskCtx* ctx = get_one_tablet_ctx(tablet_id);
if (OB_ISNULL(ctx)) {
ret = OB_ERR_NULL_VALUE;
@ -1168,7 +1171,7 @@ bool ObTenantTabletTTLMgr::can_schedule_tenant(const ObTTLTenantInfo &tenant_inf
bool ObTenantTabletTTLMgr::can_schedule_task(const ObTTLTaskCtx &ttl_task)
{
return ttl_task.task_status_ == OB_TTL_TASK_PENDING;
return !is_paused_ && ttl_task.task_status_ == OB_TTL_TASK_PENDING;
}
int ObTenantTabletTTLMgr::try_schedule_remaining_tasks(const ObTTLTaskCtx *current_ctx)
@ -1220,6 +1223,7 @@ int ObTenantTabletTTLMgr::try_schedule_task(ObTTLTaskCtx* ctx)
LOG_WARN("fail to generate dag task", KR(ret));
}
} else {
inc_dag_ref();
if (ctx->task_start_time_ == OB_INVALID_ID) {
ctx->task_start_time_ = ObTimeUtility::current_time();
}
@ -1358,5 +1362,19 @@ int ObTenantTabletTTLMgr::check_schema_version()
return ret;
}
int ObTenantTabletTTLMgr::safe_to_destroy(bool &is_safe)
{
int ret = OB_SUCCESS;
is_safe = true;
int64_t dag_ref = get_dag_ref();
if (0 != dag_ref) {
if (REACH_TIME_INTERVAL(60L * 1000000)) { // 60s
LOG_WARN("tenant ttl tablet mgr can't destroy", K(dag_ref));
}
is_safe = false;
}
return ret;
}
} // table
} // oceanbase

21
src/observer/table/ttl/ob_tenant_tablet_ttl_mgr.h
View File

@ -67,6 +67,7 @@ public:
virtual ~OBTTLTimerPeriodicTask() {}
virtual void runTimerTask() override;
private:
static const int64_t TTL_TIME_TASKER_THRESHOLD = 30 * 1000 * 1000; // 30s
ObTenantTabletTTLMgr &tablet_ttl_mgr_;
};
@ -77,10 +78,10 @@ class ObTenantTabletTTLMgr : public logservice::ObIReplaySubHandler,
public:
friend ObTTLTaskCtx;
ObTenantTabletTTLMgr()
: tenant_id_(common::OB_INVALID_TENANT_ID),
: allocator_(ObMemAttr(MTL_ID(), "TenantTTLMgr")),
tenant_id_(common::OB_INVALID_TENANT_ID),
schema_service_(NULL),
sql_proxy_(NULL),
allocator_(ObMemAttr(MTL_ID(), "TenantTTLMgr")),
is_inited_(false),
is_timer_start_(false),
periodic_delay_(TTL_PERIODIC_DELAY),
@ -89,7 +90,8 @@ public:
tg_id_(0),
local_schema_version_(OB_INVALID_VERSION),
has_start_(false),
is_paused_(false)
is_paused_(false),
dag_ref_cnt_(0)
{
}
@ -137,7 +139,8 @@ public:
int reload_tenant_task();
int report_task_status(ObTTLTaskInfo& task_info,
table::ObTTLTaskParam& task_para,
bool& is_stop);
bool& is_stop,
bool need_copy_task = true);
void on_schema_changed(uint64_t schema_changed_tenant_id);
// timer handle function
@ -145,6 +148,11 @@ public:
int check_and_handle_event();
int check_tenant_memory();
int check_inner_stat();
void inc_dag_ref() { ATOMIC_INC(&dag_ref_cnt_); }
void dec_dag_ref() { ATOMIC_DEC(&dag_ref_cnt_); }
int64_t get_dag_ref() const { return ATOMIC_LOAD(&dag_ref_cnt_); }
int safe_to_destroy(bool &is_safe);
int sync_all_dirty_task(common::ObIArray<ObTabletID>& dirty_tasks);
private:
typedef common::hash::ObHashMap<ObTabletID, ObTTLTaskCtx*> TabletTaskMap;
typedef TabletTaskMap::iterator tablet_task_iter;
@ -249,7 +257,6 @@ private:
int check_schema_version();
void resume();
void pause();
private:
static const int64_t DEFAULT_TTL_BUCKET_NUM = 100;
static const int64_t TTL_PERIODIC_DELAY = 5*1000*1000; //5s
@ -257,11 +264,12 @@ private:
static const int64_t DEFAULT_TABLE_ARRAY_SIZE = 200;
static const int64_t DEFAULT_TABLET_PAIR_SIZE = 1024;
static const int64_t DEFAULT_PARAM_BUCKET_SIZE = 200;
static const int64_t TTL_NORMAL_TIME_THRESHOLD = 3*1000*1000; // 3s
common::ObArenaAllocator allocator_;
uint64_t tenant_id_;
ObTTLTenantInfo local_tenant_task_;
share::schema::ObMultiVersionSchemaService *schema_service_;
common::ObMySQLProxy *sql_proxy_;
common::ObArenaAllocator allocator_;
bool is_inited_;
bool is_timer_start_;
int64_t periodic_delay_;
@ -273,6 +281,7 @@ private:
int64_t local_schema_version_;
bool has_start_;
bool is_paused_;
volatile int64_t dag_ref_cnt_; // ttl dag ref count for current ls
};
} // end namespace table

6
src/storage/ls/ob_ls.cpp
View File

@ -597,6 +597,7 @@ int ObLS::stop_()
}
ls_migration_handler_.stop();
ls_tablet_svr_.stop();
tablet_ttl_mgr_.stop();
is_stopped_ = true;
if (OB_SUCC(ret)) {
@ -687,6 +688,7 @@ bool ObLS::safe_to_destroy()
bool is_dup_table_handler_safe = false;
bool is_log_handler_safe = false;
bool is_transfer_handler_safe = false;
bool is_ttl_mgr_safe = false;
if (OB_FAIL(ls_tablet_svr_.safe_to_destroy(is_tablet_service_safe))) {
LOG_WARN("ls tablet service check safe to destroy failed", K(ret), KPC(this));
@ -706,6 +708,9 @@ bool ObLS::safe_to_destroy()
} else if (OB_FAIL(transfer_handler_.safe_to_destroy(is_transfer_handler_safe))) {
LOG_WARN("transfer_handler_ check safe to destroy failed", K(ret), KPC(this));
} else if (!is_transfer_handler_safe) {
} else if (OB_FAIL(tablet_ttl_mgr_.safe_to_destroy(is_ttl_mgr_safe))) {
LOG_WARN("tablet_ttl_mgr_ check safe to destroy failed", K(ret), KPC(this));
} else if (!is_ttl_mgr_safe) {
} else {
if (1 == ref_mgr_.get_total_ref_cnt()) { // only has one ref at the safe destroy task
is_safe = true;
@ -721,6 +726,7 @@ bool ObLS::safe_to_destroy()
K(is_dup_table_handler_safe),
K(is_ls_restore_handler_safe), K(is_log_handler_safe),
K(is_transfer_handler_safe),
K(is_ttl_mgr_safe),
"ls_ref", ref_mgr_.get_total_ref_cnt(),
K(ret), KP(this), KPC(this));
ref_mgr_.print();