From a7c5fc25098cc896e844eb83109bc22698f8e95b Mon Sep 17 00:00:00 2001
From: Hongqin-Li <ihongqinli@gmail.com>
Date: Tue, 21 Mar 2023 17:11:17 +0000
Subject: [PATCH] Fix ddl kv mgr deadlock

---
 src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp | 8 ++++++--
 src/storage/ddl/ob_tablet_ddl_kv_mgr.h   | 1 +
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
index 4d5b191671..27e605d7e6 100644
--- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
+++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
@@ -902,7 +902,6 @@ int ObTabletDDLKvMgr::get_active_ddl_kv_impl(ObTableHandleV2 &kv_handle)
 int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &start_scn, const SCN &scn, ObTableHandleV2 &kv_handle)
 {
   int ret = OB_SUCCESS;
-  const int64_t TRY_LOCK_TIMEOUT = 1 * 1000000; // 1s
   kv_handle.reset();
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
@@ -1105,16 +1104,21 @@ int ObTabletDDLKvMgr::get_ddl_kvs(const bool frozen_only, ObTablesHandleArray &k
 int ObTabletDDLKvMgr::get_ddl_kvs_for_query(ObTablet &tablet, ObTablesHandleArray &kv_handle_array)
 {
   int ret = OB_SUCCESS;
+  uint32_t lock_tid = 0;
   kv_handle_array.reset();
-  ObLatchRGuard guard(lock_, ObLatchIds::TABLET_DDL_KV_MGR_LOCK);
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
     LOG_WARN("ObTabletDDLKvMgr is not inited", K(ret));
+  } else if (OB_FAIL(rdlock(TRY_LOCK_TIMEOUT, lock_tid))) {
+    LOG_WARN("failed to rdlock", K(ret), K(tablet.get_tablet_meta()));
   } else if (!can_schedule_major_compaction_nolock(tablet.get_tablet_meta())) {
     // do nothing
   } else if (OB_FAIL(get_ddl_kvs_unlock(true/*frozen_only*/, kv_handle_array))) {
     LOG_WARN("get ddl kv unlock failed", K(ret));
   }
+  if (0 != lock_tid) {
+    unlock(lock_tid);
+  }
   return ret;
 }
 
diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h
index a09aed4979..0837fddd2f 100644
--- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h
+++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h
@@ -99,6 +99,7 @@ private:
   bool is_commit_success_unlock() const;
 public:
   static const int64_t MAX_DDL_KV_CNT_IN_STORAGE = 16;
+  static const int64_t TRY_LOCK_TIMEOUT = 1 * 1000000; // 1s
 private:
   bool is_inited_;
   share::SCN success_start_scn_;