From d3ea779bdc1127846849a2f6b41810fd07fb2cd7 Mon Sep 17 00:00:00 2001
From: Hongqin-Li <ihongqinli@gmail.com>
Date: Mon, 20 Feb 2023 03:11:34 +0000
Subject: [PATCH] Fix ddl kv mgr deadlock(3)

---
 src/storage/ddl/ob_ddl_redo_log_writer.cpp |  8 ++++----
 src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp   | 19 +++++++++++++++----
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp
index 7d5ccbc67..9d8a58cdd 100644
--- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp
+++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp
@@ -608,7 +608,7 @@ int ObDDLRedoLogWriter::write(
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid arguments", K(ret), K(log), K(ls_id), K(tenant_id), KP(buffer));
   } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->rdlock(ObDDLRedoLogHandle::DDL_REDO_LOG_TIMEOUT, lock_tid))) {
-    LOG_WARN("failed to wrlock", K(ret));
+    LOG_WARN("failed to rdlock", K(ret));
   } else if (ddl_kv_mgr_handle.get_obj()->get_commit_scn_nolock(tablet_handle.get_obj()->get_tablet_meta()).is_valid_and_not_min()) {
     ret = OB_TRANS_COMMITED;
     LOG_WARN("already commit", K(ret));
@@ -640,6 +640,9 @@ int ObDDLRedoLogWriter::write(
     handle.scn_ = scn;
     LOG_INFO("submit ddl redo log succeed", K(lsn), K(base_scn), K(scn));
   }
+  if (0 != lock_tid) {
+    ddl_kv_mgr_handle.get_obj()->unlock(lock_tid);
+  }
   if (OB_SUCC(ret)) {
     int64_t real_sleep_us = 0;
     if (OB_FAIL(ObDDLCtrlSpeedHandle::get_instance().limit_and_sleep(tenant_id, ls_id, buffer_size, task_id, real_sleep_us))) {
@@ -652,9 +655,6 @@ int ObDDLRedoLogWriter::write(
       cb = nullptr;
     }
   }
-  if (0 != lock_tid) {
-    ddl_kv_mgr_handle.get_obj()->unlock(lock_tid);
-  }
   return ret;
 }
 
diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
index a2e756457..371968cc1 100644
--- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
+++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
@@ -886,6 +886,7 @@ int ObTabletDDLKvMgr::get_active_ddl_kv_impl(ObTableHandleV2 &kv_handle)
 int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &start_scn, const SCN &scn, ObTableHandleV2 &kv_handle)
 {
   int ret = OB_SUCCESS;
+  const int64_t TRY_LOCK_TIMEOUT = 1 * 1000000; // 1s
   kv_handle.reset();
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
@@ -894,17 +895,24 @@ int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &start_scn, const SCN &scn,
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", K(ret), K(scn));
   } else {
-    ObLatchRGuard guard(lock_, ObLatchIds::TABLET_DDL_KV_MGR_LOCK);
-    if (start_scn != start_scn_) {
+    uint32_t lock_tid = 0; // try lock to avoid hang in clog callback
+    if (OB_FAIL(rdlock(TRY_LOCK_TIMEOUT, lock_tid))) {
+      LOG_WARN("failed to rdlock", K(ret), K(start_scn), KPC(this));
+    } else if (start_scn != start_scn_) {
       ret = OB_TASK_EXPIRED;
       LOG_WARN("ddl task expired", K(ret), K(start_scn), KPC(this));
     } else {
       try_get_ddl_kv_unlock(scn, kv_handle);
     }
+    if (lock_tid != 0) {
+      unlock(lock_tid);
+    }
   }
   if (OB_SUCC(ret) && !kv_handle.is_valid()) {
-    ObLatchWGuard guard(lock_, ObLatchIds::TABLET_DDL_KV_MGR_LOCK);
-    if (start_scn != start_scn_) {
+    uint32_t lock_tid = 0; // try lock to avoid hang in clog callback
+    if (OB_FAIL(wrlock(TRY_LOCK_TIMEOUT, lock_tid))) {
+      LOG_WARN("failed to wrlock", K(ret), K(start_scn), KPC(this));
+    } else if (start_scn != start_scn_) {
       ret = OB_TASK_EXPIRED;
       LOG_WARN("ddl task expired", K(ret), K(start_scn), KPC(this));
     } else {
@@ -915,6 +923,9 @@ int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &start_scn, const SCN &scn,
         LOG_WARN("create ddl kv failed", K(ret));
       }
     }
+    if (lock_tid != 0) {
+      unlock(lock_tid);
+    }
   }
   return ret;
 }