From 0ca997cb54d045ed2abf73a47cc5bf0c17c8c9a2 Mon Sep 17 00:00:00 2001
From: windye <yeheng_2011@163.com>
Date: Fri, 25 Aug 2023 02:44:10 +0000
Subject: [PATCH] fix core when print replay_tablet_task and remove tenant

---
 .../mtlenv/storage/blocksstable/test_index_tree.cpp  |  3 ++-
 src/observer/omt/ob_multi_tenant.cpp                 | 12 +-----------
 src/observer/omt/ob_tenant.cpp                       | 12 ++++++++++++
 src/share/rc/ob_tenant_base.cpp                      |  1 -
 .../slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp  |  4 ++--
 5 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp b/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp
index 7db02cf92..dbb3c43d9 100644
--- a/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp
+++ b/mittest/mtlenv/storage/blocksstable/test_index_tree.cpp
@@ -160,7 +160,8 @@ void TestIndexTree::TearDown()
   row_generate_.reset();
   index_row_generate_.reset();
   table_schema_.reset();
-  tenant_base_.destroy(); // stop threads
+  tenant_base_.destroy_mtl_module(); // stop threads
+  tenant_base_.destroy();
   share::ObTenantEnv::set_tenant(nullptr);
 }
 
diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp
index b30f136a6..dd6ea034c 100644
--- a/src/observer/omt/ob_multi_tenant.cpp
+++ b/src/observer/omt/ob_multi_tenant.cpp
@@ -1576,20 +1576,10 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &remove_tenant_s
     }
   }
 
-  if (OB_SUCC(ret)) {
-    if (OB_FAIL(OB_TMP_FILE_STORE.free_tenant_file_store(tenant_id))) {
-      if (OB_ENTRY_NOT_EXIST == ret) {
-        ret = OB_SUCCESS;
-      } else {
-        STORAGE_LOG(WARN, "fail to free tmp tenant file store", K(ret), K(tenant_id));
-      }
-    }
-  }
-
   if (OB_SUCC(ret) && OB_NOT_NULL(removed_tenant)) {
     ObLDHandle handle;
     if (OB_FAIL(removed_tenant->try_wait())) {
-      LOG_WARN("remove tenant try_wait failed", K(ret));
+      LOG_WARN("remove tenant try_wait failed", K(ret), K(tenant_id));
     } else if (OB_FAIL(removed_tenant->try_wrlock(handle))) {
       LOG_WARN("can't get tenant wlock to remove tenant", K(ret), K(tenant_id),
           KP(removed_tenant), K(removed_tenant->lock_));
diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp
index 15d4c24ae..51caad67a 100644
--- a/src/observer/omt/ob_tenant.cpp
+++ b/src/observer/omt/ob_tenant.cpp
@@ -966,6 +966,18 @@ void ObTenant::destroy()
   }
   group_map_.destroy_group();
   ObTenantSwitchGuard guard(this);
+  destroy_mtl_module();
+  // 1.some mtl module(eg: ObDataAccessService) remove tmp file when destroy,
+  //   so free_tenant_file_store must be after destroy_mtl_module.
+  // 2.there is tg in ObTmpTenantMemBlockManager, so free_tenant_file_store must be before
+  //   ObTenantBase::destroy() in which tg leak is checked.
+  if (OB_TMP_FAIL(OB_TMP_FILE_STORE.free_tenant_file_store(id_))) {
+    if (OB_ENTRY_NOT_EXIST == tmp_ret) {
+      tmp_ret = OB_SUCCESS;
+    } else {
+      LOG_WARN_RET(tmp_ret, "fail to free tmp tenant file store", K(ret), K_(id));
+    }
+  }
   ObTenantBase::destroy();
 
   if (nullptr != multi_level_queue_) {
diff --git a/src/share/rc/ob_tenant_base.cpp b/src/share/rc/ob_tenant_base.cpp
index ec5a6f665..89b8a8b0d 100644
--- a/src/share/rc/ob_tenant_base.cpp
+++ b/src/share/rc/ob_tenant_base.cpp
@@ -236,7 +236,6 @@ void ObTenantBase::wait_mtl_module()
 
 void ObTenantBase::destroy()
 {
-  destroy_mtl_module();
   if (tg_set_.size() > 0) {
     TGSetDumpFunc tg_set_dump_func;
     tg_set_.foreach_refactored(tg_set_dump_func);
diff --git a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp
index 1f84428f8..7c3065261 100755
--- a/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp
+++ b/src/storage/slog_ckpt/ob_tenant_checkpoint_slog_handler.cpp
@@ -785,7 +785,9 @@ int ObTenantCheckpointSlogHandler::add_replay_create_tablet_task(ObReplayCreateT
 {
   int ret = OB_SUCCESS;
   bool need_retry = false;
+  FLOG_INFO("add replay tablet task", KPC(task), K(inflight_replay_tablet_task_cnt_));
   do {
+    need_retry = false;
     if (OB_FAIL(ATOMIC_LOAD(&replay_create_tablet_errcode_))) {
       LOG_WARN("ObReplayCreateTabletTask has failed", K(ret), K(inflight_replay_tablet_task_cnt_));
     } else if (OB_FAIL(SERVER_STARTUP_TASK_HANDLER.push_task(task))) {
@@ -796,8 +798,6 @@ int ObTenantCheckpointSlogHandler::add_replay_create_tablet_task(ObReplayCreateT
       } else {
         LOG_WARN("fail to push task", K(ret), KPC(task), K(inflight_replay_tablet_task_cnt_));
       }
-    } else {
-      FLOG_INFO("add task success", KPC(task), K(inflight_replay_tablet_task_cnt_));
     }
   } while(OB_FAIL(ret) && need_retry);