fix cache dead lock

src/share/cache/ob_kvcache_store.cpp

@@ -590,7 +590,10 @@ int ObKVCacheStore::try_flush_washable_mb(
               }
             }
           }
-          de_handle_ref(handle, false /* do_retire */);
+          if (de_handle_ref(handle, false /* do_retire */) == 0) {
+            can_try_wash = false;
+            retire_list.push(&handle->retire_link_);
+          }
         }
         if (can_try_wash) {
           void *buf = nullptr;
@@ -757,6 +760,7 @@ int ObKVCacheStore::print_tenant_memblock_info(ObDLink* head)
     CREATE_WITH_TEMP_CONTEXT(param) {
       static const int64_t BUFLEN = 1 << 18;
       char *buf = (char *)ctxalp(BUFLEN);
+      HazardList retire_list;
       if (nullptr == buf) {
         ret = OB_ALLOCATE_MEMORY_FAILED;
         COMMON_LOG(WARN, "Fail to allocate memory for print tenant memblock info", K(ret), KP(buf));
@@ -778,7 +782,9 @@ int ObKVCacheStore::print_tenant_memblock_info(ObDLink* head)
                 handle->score_))) {
               COMMON_LOG(WARN, "Fail to print tenant memblock info", K(ret), K(ctx_pos));
             }
-            de_handle_ref(handle, false /* do_retire */);
+            if (de_handle_ref(handle, false /* do_retire */) == 0) {
+              retire_list.push(&handle->retire_link_);
+            }
           }
           handle = static_cast<ObKVMemBlockHandle *>(link_next(handle));
         }
@@ -786,6 +792,7 @@ int ObKVCacheStore::print_tenant_memblock_info(ObDLink* head)
           _OB_LOG(WARN, "[CACHE] len: %8ld tenant sync wash failed, cache memblock info: \n%s", ctx_pos, buf);
         }
       }
+      retire_mb_handles(retire_list, false /* do retire */);
     }
   }
   return ret;
@@ -957,7 +964,7 @@ int ObKVCacheStore::alloc_mbhandle(
   return ret;
 }
 
-void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire)
+uint32_t ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire)
 {
   int ret = OB_SUCCESS;
   uint32_t ref_cnt = 0;
@@ -971,6 +978,7 @@ void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_
       COMMON_LOG(WARN, "free_mbhandle failed", K(ret));
     }
   }
+  return ref_cnt;
 }
 
 
@@ -1413,13 +1421,15 @@ int ObKVCacheStore::remove_mb_handle(ObKVMemBlockHandle *mb_handle, const bool d
   } else {
     if (do_retire) {
       // default
-      QClockGuard guard(get_qclock());
+      {
-      dl_del(mb_handle);
+        QClockGuard guard(get_qclock());
+        dl_del(mb_handle);
+      }
+      retire_mb_handle(mb_handle, do_retire);
     } else {
       // sync wash has already get qclock
       dl_del(mb_handle);
     }
-    retire_mb_handle(mb_handle, do_retire);
   }
   return ret;
 }

src/share/cache/ob_kvcache_store.h

@@ -45,7 +45,7 @@ public:
       const enum ObKVCachePolicy policy = LRU);
 protected:
   virtual bool add_handle_ref(MBWrapper *mb_wrapper) = 0;
-  virtual void de_handle_ref(MBWrapper *mb_wrapper, const bool do_retire = true) = 0;
+  virtual uint32_t de_handle_ref(MBWrapper *mb_wrapper, const bool do_retire = true) = 0;
   virtual int alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy,
       const int64_t block_size, MBWrapper *&mb_wrapper) = 0;
   virtual int free(MBWrapper *mb_wrapper) = 0;
@@ -63,7 +63,7 @@ private:
       const enum ObKVCachePolicy policy);
 };
 
-class ObKVCacheStore : public ObIKVCacheStore<ObKVMemBlockHandle>,
+class ObKVCacheStore final : public ObIKVCacheStore<ObKVMemBlockHandle>,
     public ObIMBHandleAllocator
 {
 public:
@@ -97,7 +97,7 @@ public:
 
   virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle, const uint32_t seq_num);
   virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle);
-  virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true);
+  virtual uint32_t de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) override;
   int64_t get_handle_ref_cnt(const ObKVMemBlockHandle *mb_handle);
   virtual int64_t get_block_size() const { return block_size_; }
   // implement functions of ObIMBWrapperMgr

src/share/cache/ob_kvcache_struct.h

@@ -267,7 +267,7 @@ public:
 
   virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle, const uint32_t seq_num) = 0;
   virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle) = 0;
-  virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) = 0;
+  virtual uint32_t de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) = 0;
 
   virtual int64_t get_block_size() const = 0;
 };

src/share/cache/ob_working_set_mgr.cpp

@@ -142,11 +142,13 @@ bool ObWorkingSet::add_handle_ref(WorkingSetMB *ws_mb)
   return added;
 }
 
-void ObWorkingSet::de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire)
+uint32_t ObWorkingSet::de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire)
 {
+  uint32_t ref_cnt = 0;
   if (NULL != ws_mb) {
-    mb_handle_allocator_->de_handle_ref(ws_mb->mb_handle_, do_retire);
+    ref_cnt = mb_handle_allocator_->de_handle_ref(ws_mb->mb_handle_, do_retire);
   }
+  return ref_cnt;
 }
 
 int ObWorkingSet::alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy,

src/share/cache/ob_working_set_mgr.h

@@ -74,7 +74,7 @@ public:
 
   // implemnt functions of ObIKVCacheStore<WorkingSetMB>
   virtual bool add_handle_ref(WorkingSetMB *ws_mb);
-  virtual void de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire);
+  virtual uint32_t de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire);
   virtual int alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy,
       const int64_t block_size, WorkingSetMB *&ws_mb);
   virtual int free(WorkingSetMB *ws_mb);