fix bug: register interrupt with guard not has a return code

src/share/interrupt/ob_global_interrupt_call.cpp

@@ -140,6 +140,11 @@ int ObGlobalInterruptManager::register_checker(ObInterruptChecker *checker,
     } while (ret == OB_HASH_NOT_EXIST);
     if (OB_SUCC(ret)) {
       ATOMIC_INC(&(checker->ref_count_));
+    } else {
+      LOG_WARN("failed to register_checker");
+      if (OB_NOT_NULL(checker_node)) {
+        ob_delete(checker_node);
+      }
     }
   }
   return ret;
@@ -159,7 +164,7 @@ int ObGlobalInterruptManager::unregister_checker(ObInterruptChecker *checker,
     LIB_LOG(ERROR, "invaild checker pointer");
   } else {
     ObInterruptGetCheckerNodeCall get_node_call(checker);
-    if (OB_HASH_NOT_EXIST == map_.read_atomic(tid, get_node_call)) {
+    if (OB_HASH_NOT_EXIST == (ret = map_.read_atomic(tid, get_node_call))) {
       LIB_LOG(ERROR, "unregister checker failed", K(ret));
     } else if (!get_node_call.is_checker_exist()) {
       ret = OB_HASH_NOT_EXIST;

src/sql/engine/px/ob_px_interruption.cpp

@@ -25,12 +25,14 @@ OB_SERIALIZE_MEMBER(ObPxInterruptID, query_interrupt_id_, px_interrupt_id_);
 ObPxInterruptGuard::ObPxInterruptGuard(const ObInterruptibleTaskID &interrupt_id)
 {
   interrupt_id_ = interrupt_id;
-  SET_INTERRUPTABLE(interrupt_id_);
+  interrupt_reg_ret_ = SET_INTERRUPTABLE(interrupt_id_);
 }
 
 ObPxInterruptGuard::~ObPxInterruptGuard()
 {
+  if (OB_SUCCESS == interrupt_reg_ret_) {
     UNSET_INTERRUPTABLE(interrupt_id_);
+  }
 }
 
 int ObInterruptUtil::broadcast_px(ObIArray<ObDfo *> &dfos, int int_code)

src/sql/engine/px/ob_px_interruption.h

@@ -55,8 +55,10 @@ class ObPxInterruptGuard
 public:
   ObPxInterruptGuard(const common::ObInterruptibleTaskID &interrupt_id_);
   ~ObPxInterruptGuard();
+  int get_interrupt_reg_ret() { return interrupt_reg_ret_; }
 private:
   common::ObInterruptibleTaskID interrupt_id_;
+  int interrupt_reg_ret_;
 };
 
 

src/sql/engine/px/ob_px_rpc_processor.cpp

@@ -73,11 +73,15 @@ int ObInitSqcP::process()
    */
   if (OB_NOT_NULL(sqc_handler)) {
     ObPxRpcInitSqcArgs &arg = sqc_handler->get_sqc_init_arg();
-    SET_INTERRUPTABLE(arg.sqc_.get_interrupt_id().px_interrupt_id_);
+    if (OB_FAIL(SET_INTERRUPTABLE(arg.sqc_.get_interrupt_id().px_interrupt_id_))) {
+      LOG_WARN("sqc failed to SET_INTERRUPTABLE");
+    } else {
       unregister_interrupt_ = true;
     }
+  }
 
-  if (OB_ISNULL(sqc_handler)) {
+  if (OB_FAIL(ret)) {
+  } else if (OB_ISNULL(sqc_handler)) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("Sqc handler can't be nullptr", K(ret));
   } else if (OB_FAIL(sqc_handler->init_env())) {
@@ -408,6 +412,9 @@ int ObInitFastSqcP::process()
     ObPxRpcInitSqcArgs &arg = sqc_handler->get_sqc_init_arg();
     arg.sqc_.set_task_count(1);
     ObPxInterruptGuard px_int_guard(arg.sqc_.get_interrupt_id().px_interrupt_id_);
+    if (OB_FAIL(px_int_guard.get_interrupt_reg_ret())) {
+      LOG_WARN("fast sqc failed to SET_INTERRUPTABLE");
+    } else {
       lib::CompatModeGuard g(session->get_compatibility_mode() == ORACLE_MODE ?
       lib::Worker::CompatMode::ORACLE : lib::Worker::CompatMode::MYSQL);
       sqc_handler->set_tenant_id(session->get_effective_tenant_id());
@@ -419,6 +426,7 @@ int ObInitFastSqcP::process()
         LOG_WARN("fail to startup normal sqc", K(ret));
       }
     }
+  }
 
   //
   if (OB_SUCCESS != ret && is_schema_error(ret) && OB_NOT_NULL(sqc_handler)) {

src/sql/engine/px/ob_px_worker.cpp

@@ -163,7 +163,9 @@ void PxWorkerFunctor::operator ()()
   const bool enable_trace_log = lib::is_trace_log_enabled();
   //ensure PX worker skip updating timeout_ts_ by ntp offset
   THIS_WORKER.set_ntp_offset(0);
-  if (OB_NOT_NULL(sqc_handler) && OB_LIKELY(!sqc_handler->has_interrupted())) {
+  if (OB_FAIL(px_int_guard.get_interrupt_reg_ret())) {
+    LOG_WARN("px worker failed to SET_INTERRUPTABLE");
+  } else if (OB_NOT_NULL(sqc_handler) && OB_LIKELY(!sqc_handler->has_interrupted())) {
     THIS_WORKER.set_worker_level(sqc_handler->get_rpc_level());
     THIS_WORKER.set_curr_request_level(sqc_handler->get_rpc_level());
     LOG_TRACE("init flt ctx", K(sqc_handler->get_flt_ctx()));

unittest/share/interrupt/CMakeLists.txt

@@ -1,2 +1 @@
-##ob_unittest(test_interrupt_system.cpp)
+ob_unittest(test_interrupt_system)
-

unittest/share/interrupt/test_interrupt_system.cpp

@@ -15,105 +15,192 @@
 
 #define private public
 
-#include "lib/coro/testing.h"
 #include "share/interrupt/ob_interrupt_rpc_proxy.h"
 #include "share/interrupt/ob_global_interrupt_call.h"
+#include "sql/engine/px/ob_px_interruption.h"
 #include "rpc/testing.h"
+#include <thread>
 
 using namespace oceanbase::obrpc;
 
-namespace oceanbase {
+namespace oceanbase
-namespace common {
+{
-
+namespace common
-TEST(ObGlobalInterruptManager, normal)
 {
 
-  rpctesting::Service service;
+rpctesting::Service g_service;
-  ObInterruptProcessor p;
+ObInterruptProcessor g_processor;
-  service.init();
+ObInterruptRpcProxy g_proxy;
-  service.reg_processor(&p);
+ObGlobalInterruptManager *g_manager(nullptr);
-  ObInterruptRpcProxy proxy;
-  service.get_proxy(proxy);
 
+void init_rpc()
+{
   int ret = OB_SUCCESS;
-  int temp_id = 1;
+  ret = g_service.init();
-  int icode = 1;
+  ASSERT_EQ(OB_SUCCESS, ret);
+
+  ret = g_service.reg_processor(&g_processor);
+  ASSERT_EQ(OB_SUCCESS, ret);
+
+  ret = g_service.get_proxy(g_proxy);
+  ASSERT_EQ(OB_SUCCESS, ret);
+}
+
+TEST(ObGlobalInterruptManager, init_manager)
+{
+  int ret = OB_SUCCESS;
+  init_rpc();
   ObAddr local(ObAddr::IPV4, "127.0.0.1", 10086);
-  ObAddr dst = service.get_dst();
+  ObAddr dst = g_service.get_dst();
-
-  /// Singleton acquisition
-  ObGlobalInterruptManager *manager = ObGlobalInterruptManager::getInstance();
-  ASSERT_EQ(false, nullptr == manager);
 
+  // prepare mocked ObInterruptibleTaskID, ObInterruptCode,
+  ObInterruptibleTaskID interrupt_id(1, 1);
+  ObInterruptStackInfo interrupt_stack_info;
+  interrupt_stack_info.set_info(1 /*from tid*/, local /*from_svr_addr*/,
+                                "mock error" /*extra_msg*/);
+  ObInterruptCode interrupt_code(OB_RPC_CONNECT_ERROR, interrupt_stack_info);
   ObInterruptChecker *checker = nullptr;
-  /// Perform various operations without initialization, correct feedback
 
-  ret = manager->interrupt(temp_id, icode);
+  // Singleton acquisition
+  g_manager = ObGlobalInterruptManager::getInstance();
+  ASSERT_EQ(false, nullptr == g_manager);
+
+  // test init ObGlobalInterruptManager
+  ret = g_manager->interrupt(interrupt_id, interrupt_code);
   ASSERT_EQ(OB_NOT_INIT, ret);
 
-  ret = manager->interrupt(local, temp_id, icode);
+  ret = g_manager->interrupt(local, interrupt_id, interrupt_code);
   ASSERT_EQ(OB_NOT_INIT, ret);
 
-  ret = manager->register_checker(checker);
+  ret = g_manager->register_checker(checker, interrupt_id);
   ASSERT_EQ(OB_NOT_INIT, ret);
 
-  ret = manager->unregister_checker(checker);
+  ret = g_manager->unregister_checker(checker, interrupt_id);
   ASSERT_EQ(OB_NOT_INIT, ret);
 
-  /// No RPC initialization is illegal
+  // No RPC initialization is illegal
-  ret = manager->init(local, nullptr);
+  ret = g_manager->init(local, nullptr);
   ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
 
-  /// Regular initialization
+  // Regular initialization
-  ret = manager->init(local, &proxy);
+  ret = g_manager->init(local, &g_proxy);
   ASSERT_EQ(OB_SUCCESS, ret);
 
-  /// Initialize correctly and feedback error many times
+  // Initialize correctly and feedback error many times
-  ret = manager->init(local, &proxy);
+  ret = g_manager->init(local, &g_proxy);
   ASSERT_EQ(OB_INIT_TWICE, ret);
+}
 
-  ret = manager->register_checker(checker);
+TEST(ObGlobalInterruptManager, register_and_unregister)
-  ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
+{
+  int ret = OB_SUCCESS;
+  ObAddr local(ObAddr::IPV4, "127.0.0.1", 10086);
+  ObAddr dst = g_service.get_dst();
 
-  ret = manager->unregister_checker(checker);
+  // prepare mocked ObInterruptibleTaskID, ObInterruptCode,
-  ASSERT_EQ(OB_INVALID_ARGUMENT, ret);
+  ObInterruptibleTaskID interrupt_id(1, 1);
+  ObInterruptStackInfo interrupt_stack_info;
+  interrupt_stack_info.set_info(1 /*from tid*/, local /*from_svr_addr*/,
+                                "mock error" /*extra_msg*/);
+  ObInterruptCode interrupt_code(OB_RPC_CONNECT_ERROR, interrupt_stack_info);
 
-  /// Get the inspector
+  // prepare checker
-  checker = new ObInterruptChecker(true, temp_id);
+  ObInterruptChecker *checker = nullptr;
-  /// Test self-registration
+  checker = new ObInterruptChecker(false, 1); // no coroutine again
-  ret = checker->register_checker(temp_id);
+
+  // Test self-registration
+  ret = checker->register_checker(interrupt_id);
   ASSERT_EQ(OB_SUCCESS, ret);
 
-  /// Test duplicate registration
+  // Test duplicate registration
-  ret = checker->register_checker(temp_id);
+  ret = checker->register_checker(interrupt_id);
-  ASSERT_EQ(OB_INIT_TWICE, ret);
+  ASSERT_EQ(OB_HASH_EXIST, ret);
-  ret = checker->register_checker(temp_id + 1);
+  ret = g_manager->register_checker(checker, interrupt_id);
-  ASSERT_EQ(OB_INIT_TWICE, ret);
+  ASSERT_EQ(OB_HASH_EXIST, ret);
-  ret = manager->register_checker(checker);
-  ASSERT_EQ(OB_INIT_TWICE, ret);
 
-  /// Semaphore detection without interruption
+  // unregister checker
+  ret = g_manager->unregister_checker(checker, interrupt_id);
+  ASSERT_EQ(OB_SUCCESS, ret);
+
+  // duplicate unregister checker
+  ret = g_manager->unregister_checker(checker, interrupt_id);
+  ASSERT_EQ(OB_HASH_NOT_EXIST, ret);
+  EXPECT_EQ(0, g_manager->map_.size());
+
+  delete checker;
+
+  // Test group registration and deregistration
+  ObInterruptChecker *checker0 = new ObInterruptChecker(false, 1);
+  ObInterruptChecker *checker1 = new ObInterruptChecker(false, 1);
+  ObInterruptChecker *checker2 = new ObInterruptChecker(false, 1);
+  ObInterruptChecker *checker3 = new ObInterruptChecker(false, 1);
+  ret = g_manager->register_checker(checker0, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->register_checker(checker1, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->register_checker(checker2, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->register_checker(checker3, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->unregister_checker(checker0, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->unregister_checker(checker1, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->unregister_checker(checker2, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  ret = g_manager->unregister_checker(checker3, interrupt_id);
+  EXPECT_EQ(OB_SUCCESS, ret);
+  delete checker0;
+  delete checker1;
+  delete checker2;
+  delete checker3;
+
+  EXPECT_EQ(0, g_manager->map_.size());
+}
+
+TEST(ObGlobalInterruptManager, interrupt)
+{
+  int ret = OB_SUCCESS;
+  ObAddr local(ObAddr::IPV4, "127.0.0.1", 10086);
+  ObAddr dst = g_service.get_dst();
+
+  // prepare mocked ObInterruptibleTaskID, ObInterruptCode,
+  ObInterruptibleTaskID interrupt_id(1, 1);
+  ObInterruptStackInfo interrupt_stack_info;
+  interrupt_stack_info.set_info(1 /*from tid*/, local /*from_svr_addr*/,
+                                "mock error" /*extra_msg*/);
+  ObInterruptCode interrupt_code(OB_RPC_CONNECT_ERROR, interrupt_stack_info);
+
+  // prepare checker && register checker
+  ObInterruptChecker *checker = nullptr;
+  checker = new ObInterruptChecker(false, 1); // no coroutine again
+  ret = checker->register_checker(interrupt_id);
+  ASSERT_EQ(OB_SUCCESS, ret);
+
+  // Semaphore detection without interruption
   bool r = checker->is_interrupted();
   ASSERT_EQ(false, r);
 
-  /// Local execution interrupt check semaphore
+  // Local execution interrupt check semaphore
-  ret = manager->interrupt(temp_id, icode);
+  ret = g_manager->interrupt(interrupt_id, interrupt_code);
   ASSERT_EQ(OB_SUCCESS, ret);
 
   r = checker->is_interrupted();
   ASSERT_EQ(true, r);
 
-  /// Release checker
+  // unregister checker
-  ret = manager->unregister_checker(checker);
+  ret = g_manager->unregister_checker(checker, interrupt_id);
   ASSERT_EQ(OB_SUCCESS, ret);
 
-  /// The interrupt can still be called after the checker is released
+  // The interrupt can still be called after the checker is unregistered
-  ret = manager->interrupt(local, temp_id, icode);
+  // but can not actually do interrupt, the return code is OB_HASH_NOT_EXIST BUT overwrite by
+  // OB_SUCCESS
+  ret = g_manager->interrupt(local, interrupt_id, interrupt_code);
   ASSERT_EQ(OB_SUCCESS, ret);
 
-  /// Downgrade to local interrupt test when remote interrupt is passed to local address
+  // Downgrade to local interrupt test when remote interrupt is passed to local address
-  manager->register_checker(checker);
+  g_manager->register_checker(checker, interrupt_id);
-  ret = manager->interrupt(local, temp_id, icode);
+  ret = g_manager->interrupt(local, interrupt_id, interrupt_code);
   ASSERT_EQ(OB_SUCCESS, ret);
 
   r = checker->is_interrupted();
@@ -124,125 +211,139 @@ TEST(ObGlobalInterruptManager, normal)
   r = checker->is_interrupted();
   ASSERT_EQ(false, r);
 
-  manager->interrupt(temp_id, icode);
+  g_manager->interrupt(interrupt_id, interrupt_code);
   r = checker->is_interrupted();
   ASSERT_EQ(true, r);
 
-  /// rpc call test
+  // rpc call test
   checker->clear_status();
-  ret = manager->interrupt(dst, temp_id, icode);
+  ret = g_manager->interrupt(dst, interrupt_id, interrupt_code);
   ASSERT_EQ(OB_SUCCESS, ret);
 
+  sleep(1);
   r = checker->is_interrupted();
   ASSERT_EQ(true, r);
 
+  // Automatic logout of test destruction
+  // unregister checker
+  ret = g_manager->unregister_checker(checker, interrupt_id);
+  ASSERT_EQ(OB_SUCCESS, ret);
 
-
+  ObInterruptCheckerNode *checker_node = nullptr;
-  //Automatic logout of test destruction
+  ret = g_manager->map_.get_refactored(interrupt_id, checker_node);
-  delete checker;
-
-  ret = manager->map_.get_refactored(temp_id, checker);
   EXPECT_EQ(OB_HASH_NOT_EXIST, ret);
 
-  // Test group registration, interruption and deregistration
+  EXPECT_EQ(0, g_manager->map_.size());
-  ObInterruptChecker *checker0 = new ObInterruptChecker(true, 1);
-  ObInterruptChecker *checker1 = new ObInterruptChecker(true, 1);
-  ObInterruptChecker *checker2 = new ObInterruptChecker(true, 1);
-  ObInterruptChecker *checker3 = new ObInterruptChecker(true, 1);
-  ret = manager->register_checker(checker0);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->register_checker(checker1);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->register_checker(checker2);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->register_checker(checker3);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->unregister_checker(checker0);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->unregister_checker(checker1);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->unregister_checker(checker2);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  ret = manager->unregister_checker(checker3);
-  EXPECT_EQ(OB_SUCCESS, ret);
-  delete checker0;
-  delete checker1;
-  delete checker2;
-  delete checker3;
 
+  delete checker;
+}
 
-  // The following is in coroutine mode
+enum RegMode
-  // Simulate regular usage scenarios
+{
-  // And test the use of several global functions
+  ManagerInterface = 0,
+  CheckerInterface = 1,
+  Guard = 2,
+  SetInterruptable = 3,
+};
 
-  int total = -1;
+static void mulit_reg_unreg(RegMode reg_mod) {
-  int fail = 0;
+  int ret = OB_SUCCESS;
-  int ready = 0;
+  ObRandom random;
-  uint64_t n[40];
+  uint64_t slp_time = abs(random.get_int32()) % 100 + 1;
-
+  ObInterruptibleTaskID interrupt_id(1, 1);
-  auto apool = cotesting::FlexPool {
+  ObInterruptChecker *checker = new ObInterruptChecker(false, 1);
-      [&] {
+  switch (reg_mod) {
-        int idx = ATOMIC_AAF(&total, 1);
+    case RegMode::ManagerInterface: {
-        uint64_t cid = CO_ID();
+      ret = g_manager->register_checker(checker, interrupt_id);
-        n[idx] = cid;
+      EXPECT_EQ(OB_SUCCESS, ret);
-        SET_INTERRUPTABLE(cid);
+      ob_usleep(slp_time);
-        /// The first 39 coroutines are waiting to be terminated
+      ret = g_manager->unregister_checker(checker, interrupt_id);
-        if (idx < 39) {
+      EXPECT_EQ(OB_SUCCESS, ret);
-          while (ready == 0) {
+      break;
-            ::usleep(1000);
     }
-          if (IS_INTERRUPTED()) {
+    case RegMode::CheckerInterface: {
-            ASSERT_EQ(idx + 10, GET_INTERRUPT_CODE());
+      ret = checker->register_checker(interrupt_id);
-            return;
+      ASSERT_EQ(OB_SUCCESS, ret);
+      ob_usleep(slp_time);
+      checker->unregister_checker(interrupt_id);
+      break;
     }
-          fail++;
+    case RegMode::Guard: {
+      sql::ObPxInterruptGuard px_int_guard(interrupt_id);
+      break;
     }
-        /// The 40th coroutine is responsible for terminating them with three scenarios respectively
+    case RegMode::SetInterruptable: {
-        else {
+      ret = SET_INTERRUPTABLE(interrupt_id);
-          for (int i = 0; i < 39; i++) {
+      ASSERT_EQ(OB_SUCCESS, ret);
-            int k = i % 3;
+      ob_usleep(slp_time);
-            if (k == 0)
+      UNSET_INTERRUPTABLE(interrupt_id);
-              manager->interrupt(n[i], i + 10);
+      break;
-            else if (k == 1)
+    }
-              manager->interrupt(local, n[i], i + 10);
+    default: {
-            else
+      ret = OB_ERR_UNEXPECTED;
-              manager->interrupt(dst, n[i], i + 10);
+      break;
     }
-          ASSERT_EQ(false, IS_INTERRUPTED());
-          ready++;
   }
-      },
-      4, 10};
-  apool.start(true);
-  ASSERT_EQ(0, fail);
 
-  int xcnt = 0;
+  delete checker;
-  // Test group interrupted
+}
-  SET_INTERRUPTABLE(10000);
 
-  auto bpool = cotesting::FlexPool{
+TEST(ObGlobalInterruptManager, multi_thread_register_unregister)
-      [&] {
+{
-        SET_INTERRUPTABLE(10000);
+  const int64_t thread_cnt = 2048;
-        ATOMIC_INC(&xcnt);
+  std::thread threads[thread_cnt];
+  for (int64_t i = 0; i < thread_cnt; ++i) {
+    RegMode reg_mod = static_cast<RegMode>(i % 4);
+    threads[i] = std::thread(mulit_reg_unreg, reg_mod);
+  }
+
+  for (int i = 0; i < thread_cnt; ++i) {
+    threads[i].join();
+  }
+
+  EXPECT_EQ(0, g_manager->map_.size());
+}
+
+static void thread_wait_interrupt(void *ready_thr_cnt)
+{
+  ObInterruptibleTaskID interrupt_id(2, 2);
+  SET_INTERRUPTABLE(interrupt_id);
+  std::atomic<int> *ready_cnt = static_cast<std::atomic<int>*> (ready_thr_cnt);
+  ready_cnt->fetch_add(1);
   while (!IS_INTERRUPTED()) {
-          CO_YIELD();
+    ob_usleep(100);
   }
-        EXPECT_EQ(1, GET_INTERRUPT_CODE());
+  UNSET_INTERRUPTABLE(interrupt_id);
-        ATOMIC_DEC(&xcnt);
+}
-      },
-      10, 10};
-  bpool.start(false);
-  // Wait for all coroutines to execute set_interruptable
-  while (xcnt != 100) {};
 
-  // Execute interrupt for taskid = 10000
+TEST(ObGlobalInterruptManager, interrupt_all_threads)
-  manager->interrupt(10000, 1);
+{
-  bpool.wait();
+  const int64_t thread_cnt = 64;
-  // Coroutine interrupt count is cleared to 0
+  std::thread threads[thread_cnt];
-  ASSERT_EQ(0, xcnt);
+  std::atomic<int> ready_thr_cnt{0};
-  // The current thread also receives an interrupt signal
+  for (int64_t i = 0; i < thread_cnt; ++i) {
-  ASSERT_EQ(true, IS_INTERRUPTED());
+    threads[i] = std::thread(thread_wait_interrupt, &ready_thr_cnt);
-  ASSERT_EQ(1, GET_INTERRUPT_CODE());
+  }
+
+  // waiting all threads ready
+  while (ready_thr_cnt.load() != thread_cnt) {
+    ob_usleep(100);
+  }
+
+  ObInterruptibleTaskID interrupt_id(2, 2);
+  ObInterruptStackInfo interrupt_stack_info;
+  ObAddr dst = g_service.get_dst();
+  interrupt_stack_info.set_info(1 /*from tid*/, dst /*from_svr_addr*/,
+                                "mock error" /*extra_msg*/);
+  ObInterruptCode interrupt_code(OB_RPC_CONNECT_ERROR, interrupt_stack_info);
+
+  g_manager->interrupt(dst, interrupt_id, interrupt_code);
+
+  for (int i = 0; i < thread_cnt; ++i) {
+    threads[i].join();
+  }
+
+  EXPECT_EQ(0, g_manager->map_.size());
 }
 
 } // namespace common
@@ -250,7 +351,8 @@ TEST(ObGlobalInterruptManager, normal)
 
 int main(int argc, char **argv)
 {
-  oceanbase::common::ObLogger::get_logger().set_log_level("DEBUG");
+  OB_LOGGER.set_file_name("test_ob_interrupt_manager.log", true, true);
+  OB_LOGGER.set_log_level("DEBUG");
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }