From 0302441f38ca7ab205c048247bcde331c45a2737 Mon Sep 17 00:00:00 2001 From: z404289981 Date: Tue, 29 Aug 2023 12:10:32 +0000 Subject: [PATCH] fix cache dead lock --- src/share/cache/ob_kvcache_store.cpp | 22 ++++++++++++++++------ src/share/cache/ob_kvcache_store.h | 6 +++--- src/share/cache/ob_kvcache_struct.h | 2 +- src/share/cache/ob_working_set_mgr.cpp | 6 ++++-- src/share/cache/ob_working_set_mgr.h | 2 +- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/share/cache/ob_kvcache_store.cpp b/src/share/cache/ob_kvcache_store.cpp index e0ad7780e4..903ec672fb 100644 --- a/src/share/cache/ob_kvcache_store.cpp +++ b/src/share/cache/ob_kvcache_store.cpp @@ -590,7 +590,10 @@ int ObKVCacheStore::try_flush_washable_mb( } } } - de_handle_ref(handle, false /* do_retire */); + if (de_handle_ref(handle, false /* do_retire */) == 0) { + can_try_wash = false; + retire_list.push(&handle->retire_link_); + } } if (can_try_wash) { void *buf = nullptr; @@ -757,6 +760,7 @@ int ObKVCacheStore::print_tenant_memblock_info(ObDLink* head) CREATE_WITH_TEMP_CONTEXT(param) { static const int64_t BUFLEN = 1 << 18; char *buf = (char *)ctxalp(BUFLEN); + HazardList retire_list; if (nullptr == buf) { ret = OB_ALLOCATE_MEMORY_FAILED; COMMON_LOG(WARN, "Fail to allocate memory for print tenant memblock info", K(ret), KP(buf)); @@ -778,7 +782,9 @@ int ObKVCacheStore::print_tenant_memblock_info(ObDLink* head) handle->score_))) { COMMON_LOG(WARN, "Fail to print tenant memblock info", K(ret), K(ctx_pos)); } - de_handle_ref(handle, false /* do_retire */); + if (de_handle_ref(handle, false /* do_retire */) == 0) { + retire_list.push(&handle->retire_link_); + } } handle = static_cast(link_next(handle)); } @@ -786,6 +792,7 @@ int ObKVCacheStore::print_tenant_memblock_info(ObDLink* head) _OB_LOG(WARN, "[CACHE] len: %8ld tenant sync wash failed, cache memblock info: \n%s", ctx_pos, buf); } } + retire_mb_handles(retire_list, false /* do retire */); } } return ret; @@ -957,7 +964,7 @@ int ObKVCacheStore::alloc_mbhandle( return ret; } -void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire) +uint32_t ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire) { int ret = OB_SUCCESS; uint32_t ref_cnt = 0; @@ -971,6 +978,7 @@ void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_ COMMON_LOG(WARN, "free_mbhandle failed", K(ret)); } } + return ref_cnt; } @@ -1413,13 +1421,15 @@ int ObKVCacheStore::remove_mb_handle(ObKVMemBlockHandle *mb_handle, const bool d } else { if (do_retire) { // default - QClockGuard guard(get_qclock()); - dl_del(mb_handle); + { + QClockGuard guard(get_qclock()); + dl_del(mb_handle); + } + retire_mb_handle(mb_handle, do_retire); } else { // sync wash has already get qclock dl_del(mb_handle); } - retire_mb_handle(mb_handle, do_retire); } return ret; } diff --git a/src/share/cache/ob_kvcache_store.h b/src/share/cache/ob_kvcache_store.h index 70eba05cb0..c22a6b13ab 100644 --- a/src/share/cache/ob_kvcache_store.h +++ b/src/share/cache/ob_kvcache_store.h @@ -45,7 +45,7 @@ public: const enum ObKVCachePolicy policy = LRU); protected: virtual bool add_handle_ref(MBWrapper *mb_wrapper) = 0; - virtual void de_handle_ref(MBWrapper *mb_wrapper, const bool do_retire = true) = 0; + virtual uint32_t de_handle_ref(MBWrapper *mb_wrapper, const bool do_retire = true) = 0; virtual int alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy, const int64_t block_size, MBWrapper *&mb_wrapper) = 0; virtual int free(MBWrapper *mb_wrapper) = 0; @@ -63,7 +63,7 @@ private: const enum ObKVCachePolicy policy); }; -class ObKVCacheStore : public ObIKVCacheStore, +class ObKVCacheStore final : public ObIKVCacheStore, public ObIMBHandleAllocator { public: @@ -97,7 +97,7 @@ public: virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle, const uint32_t seq_num); virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle); - virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true); + virtual uint32_t de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) override; int64_t get_handle_ref_cnt(const ObKVMemBlockHandle *mb_handle); virtual int64_t get_block_size() const { return block_size_; } // implement functions of ObIMBWrapperMgr diff --git a/src/share/cache/ob_kvcache_struct.h b/src/share/cache/ob_kvcache_struct.h index c9ba2dc4d7..a2391f321a 100644 --- a/src/share/cache/ob_kvcache_struct.h +++ b/src/share/cache/ob_kvcache_struct.h @@ -267,7 +267,7 @@ public: virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle, const uint32_t seq_num) = 0; virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle) = 0; - virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) = 0; + virtual uint32_t de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) = 0; virtual int64_t get_block_size() const = 0; }; diff --git a/src/share/cache/ob_working_set_mgr.cpp b/src/share/cache/ob_working_set_mgr.cpp index 2233c78e9d..61f3a14348 100644 --- a/src/share/cache/ob_working_set_mgr.cpp +++ b/src/share/cache/ob_working_set_mgr.cpp @@ -142,11 +142,13 @@ bool ObWorkingSet::add_handle_ref(WorkingSetMB *ws_mb) return added; } -void ObWorkingSet::de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire) +uint32_t ObWorkingSet::de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire) { + uint32_t ref_cnt = 0; if (NULL != ws_mb) { - mb_handle_allocator_->de_handle_ref(ws_mb->mb_handle_, do_retire); + ref_cnt = mb_handle_allocator_->de_handle_ref(ws_mb->mb_handle_, do_retire); } + return ref_cnt; } int ObWorkingSet::alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy, diff --git a/src/share/cache/ob_working_set_mgr.h b/src/share/cache/ob_working_set_mgr.h index ceffec33c9..eff9c52049 100644 --- a/src/share/cache/ob_working_set_mgr.h +++ b/src/share/cache/ob_working_set_mgr.h @@ -74,7 +74,7 @@ public: // implemnt functions of ObIKVCacheStore virtual bool add_handle_ref(WorkingSetMB *ws_mb); - virtual void de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire); + virtual uint32_t de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire); virtual int alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy, const int64_t block_size, WorkingSetMB *&ws_mb); virtual int free(WorkingSetMB *ws_mb);