fix core when print replay_tablet_task and remove tenant

mittest/mtlenv/storage/blocksstable/test_index_tree.cpp

@@ -160,7 +160,8 @@ void TestIndexTree::TearDown()
   row_generate_.reset();
   index_row_generate_.reset();
   table_schema_.reset();
-  tenant_base_.destroy(); // stop threads
+  tenant_base_.destroy_mtl_module(); // stop threads
+  tenant_base_.destroy();
   share::ObTenantEnv::set_tenant(nullptr);
 }
 

src/observer/omt/ob_multi_tenant.cpp

@@ -1576,20 +1576,10 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &remove_tenant_s
     }
   }
 
-  if (OB_SUCC(ret)) {
-    if (OB_FAIL(OB_TMP_FILE_STORE.free_tenant_file_store(tenant_id))) {
-      if (OB_ENTRY_NOT_EXIST == ret) {
-        ret = OB_SUCCESS;
-      } else {
-        STORAGE_LOG(WARN, "fail to free tmp tenant file store", K(ret), K(tenant_id));
-      }
-    }
-  }
-
   if (OB_SUCC(ret) && OB_NOT_NULL(removed_tenant)) {
     ObLDHandle handle;
     if (OB_FAIL(removed_tenant->try_wait())) {
-      LOG_WARN("remove tenant try_wait failed", K(ret));
+      LOG_WARN("remove tenant try_wait failed", K(ret), K(tenant_id));
     } else if (OB_FAIL(removed_tenant->try_wrlock(handle))) {
       LOG_WARN("can't get tenant wlock to remove tenant", K(ret), K(tenant_id),
           KP(removed_tenant), K(removed_tenant->lock_));

src/observer/omt/ob_tenant.cpp

@@ -966,6 +966,18 @@ void ObTenant::destroy()
   }
   group_map_.destroy_group();
   ObTenantSwitchGuard guard(this);
+  destroy_mtl_module();
+  // 1.some mtl module(eg: ObDataAccessService) remove tmp file when destroy,
+  //   so free_tenant_file_store must be after destroy_mtl_module.
+  // 2.there is tg in ObTmpTenantMemBlockManager, so free_tenant_file_store must be before
+  //   ObTenantBase::destroy() in which tg leak is checked.
+  if (OB_TMP_FAIL(OB_TMP_FILE_STORE.free_tenant_file_store(id_))) {
+    if (OB_ENTRY_NOT_EXIST == tmp_ret) {
+      tmp_ret = OB_SUCCESS;
+    } else {
+      LOG_WARN_RET(tmp_ret, "fail to free tmp tenant file store", K(ret), K_(id));
+    }
+  }
   ObTenantBase::destroy();
 
   if (nullptr != multi_level_queue_) {

src/share/rc/ob_tenant_base.cpp

@@ -236,7 +236,6 @@ void ObTenantBase::wait_mtl_module()
 
 void ObTenantBase::destroy()
 {
-  destroy_mtl_module();
   if (tg_set_.size() > 0) {
     TGSetDumpFunc tg_set_dump_func;
     tg_set_.foreach_refactored(tg_set_dump_func);

src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp

@@ -785,7 +785,9 @@ int ObTenantCheckpointSlogHandler::add_replay_create_tablet_task(ObReplayCreateT
 {
   int ret = OB_SUCCESS;
   bool need_retry = false;
+  FLOG_INFO("add replay tablet task", KPC(task), K(inflight_replay_tablet_task_cnt_));
   do {
+    need_retry = false;
     if (OB_FAIL(ATOMIC_LOAD(&replay_create_tablet_errcode_))) {
       LOG_WARN("ObReplayCreateTabletTask has failed", K(ret), K(inflight_replay_tablet_task_cnt_));
     } else if (OB_FAIL(SERVER_STARTUP_TASK_HANDLER.push_task(task))) {
@@ -796,8 +798,6 @@ int ObTenantCheckpointSlogHandler::add_replay_create_tablet_task(ObReplayCreateT
       } else {
         LOG_WARN("fail to push task", K(ret), KPC(task), K(inflight_replay_tablet_task_cnt_));
       }
-    } else {
-      FLOG_INFO("add task success", KPC(task), K(inflight_replay_tablet_task_cnt_));
     }
   } while(OB_FAIL(ret) && need_retry);