hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix retire deadlock when retire mb handle in sync wash

Browse Source
This commit is contained in:
obdev 2023-04-13 06:50:07 +00:00 committed by ob-robot
parent 36e28a5f79
commit f699eaf0bc
5 changed files with 41 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/share/cache/ob_kvcache_store.cpp vendored
View File

@ -112,9 +112,7 @@ void ObKVCacheStore::destroy()
}
// free all mb handles cached by threads
HazardList reclaim_list;
get_retire_station().purge(reclaim_list);
reuse_mb_handles(reclaim_list);
purge_mb_handle_retire_station();
ob_free(mb_handles_);
mb_handles_ = NULL;
@ -338,8 +336,8 @@ bool ObKVCacheStore::wash()
}
current_time = ObTimeUtility::current_time();
wash_time = current_time - start_time;
}
}
purge_mb_handle_retire_station();
COMMON_LOG(INFO, "Wash time detail, ", K(compute_wash_size_time), K(refresh_score_time), K(wash_time));
}
@ -590,7 +588,7 @@ int ObKVCacheStore::try_flush_washable_mb(
}
}
}
de_handle_ref(handle);
de_handle_ref(handle, false /* do_retire */);
}
if (can_try_wash) {
void *buf = nullptr;
@ -676,7 +674,7 @@ int ObKVCacheStore::try_flush_washable_mb(
COMMON_LOG(INFO, "ObKVCache try flush washable memblock details", K(ret), K(force_flush), K(tenant_id),
K(cache_id), K(size_washed), K(size_need_washed));
retire_mb_handles(retire_list);
retire_mb_handles(retire_list, true /* do retire */);
COMMON_LOG(DEBUG, "Try flush cache result", K(size_washed), K(size_need_washed), K(tenant_id), K(cache_id), K(ret));
}
@ -711,6 +709,13 @@ int ObKVCacheStore::inner_push_memblock_info(const ObKVMemBlockHandle &handle, O
return ret;
}
void ObKVCacheStore::purge_mb_handle_retire_station()
{
HazardList reclaim_list;
get_retire_station().purge(reclaim_list);
reuse_mb_handles(reclaim_list);
}
int ObKVCacheStore::get_memblock_info(const uint64_t tenant_id, ObIArray<ObKVCacheStoreMemblockInfo> &memblock_infos)
{
int ret = OB_SUCCESS;
@ -849,7 +854,7 @@ int ObKVCacheStore::mark_washable(ObKVMemBlockHandle *mb_handle)
return ret;
}
int ObKVCacheStore::free_mbhandle(ObKVMemBlockHandle *mb_handle)
int ObKVCacheStore::free_mbhandle(ObKVMemBlockHandle *mb_handle, const bool do_retire)
{
int ret = OB_SUCCESS;
if (!inited_) {
@ -866,7 +871,7 @@ int ObKVCacheStore::free_mbhandle(ObKVMemBlockHandle *mb_handle)
COMMON_LOG(ERROR, "do_wash_mb failed", K(ret));
} else {
free_mb(*mb_handle->inst_->mb_list_handle_.get_resource_handle(), tenant_id, buf);
if (OB_FAIL(remove_mb_handle(mb_handle))) {
if (OB_FAIL(remove_mb_handle(mb_handle, do_retire))) {
COMMON_LOG(WARN, "remove_mb failed", K(ret));
}
}
@ -950,7 +955,7 @@ int ObKVCacheStore::alloc_mbhandle(
return ret;
}
void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle)
void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire)
{
int ret = OB_SUCCESS;
uint32_t ref_cnt = 0;
@ -960,7 +965,7 @@ void ObKVCacheStore::de_handle_ref(ObKVMemBlockHandle *mb_handle)
} else if (OB_FAIL(mb_handle->handle_ref_.dec_ref_cnt_and_inc_seq_num(ref_cnt))) {
COMMON_LOG(ERROR, "Fail to dec ref cnt, ", K(ret));
} else if (0 == ref_cnt) {
if (OB_FAIL(free_mbhandle(mb_handle))) {
if (OB_FAIL(free_mbhandle(mb_handle, do_retire))) {
COMMON_LOG(WARN, "free_mbhandle failed", K(ret));
}
}
@ -1393,7 +1398,7 @@ int ObKVCacheStore::insert_mb_handle(ObDLink *head, ObKVMemBlockHandle *mb_handl
return ret;
}
int ObKVCacheStore::remove_mb_handle(ObKVMemBlockHandle *mb_handle)
int ObKVCacheStore::remove_mb_handle(ObKVMemBlockHandle *mb_handle, const bool do_retire)
{
int ret = OB_SUCCESS;
if (!inited_) {
@ -1403,30 +1408,34 @@ int ObKVCacheStore::remove_mb_handle(ObKVMemBlockHandle *mb_handle)
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments", K(ret), KP(mb_handle));
} else {
{
if (do_retire) {
// default
QClockGuard guard(get_qclock());
dl_del(mb_handle);
} else {
// sync wash has already get qclock
dl_del(mb_handle);
}
retire_mb_handle(mb_handle);
retire_mb_handle(mb_handle, do_retire);
}
return ret;
}
void ObKVCacheStore::retire_mb_handle(ObKVMemBlockHandle *mb_handle)
void ObKVCacheStore::retire_mb_handle(ObKVMemBlockHandle *mb_handle, const bool do_retire)
{
if (NULL != mb_handle) {
HazardList retire_list;
retire_list.push(&mb_handle->retire_link_);
retire_mb_handles(retire_list);
retire_mb_handles(retire_list, do_retire);
}
}
void ObKVCacheStore::retire_mb_handles(HazardList &retire_list)
void ObKVCacheStore::retire_mb_handles(HazardList &retire_list, const bool do_retire)
{
if (retire_list.size() > 0) {
HazardList reclaim_list;
int64_t retire_limit = RETIRE_LIMIT;
if (wash_itid_ == get_itid()) {
int64_t retire_limit = do_retire ? RETIRE_LIMIT : INT64_MAX;
if (wash_itid_ == get_itid()) { // wash thread should not sync wash
retire_limit = WASH_THREAD_RETIRE_LIMIT;
}
get_retire_station().retire(reclaim_list, retire_list, retire_limit);

13
src/share/cache/ob_kvcache_store.h vendored
View File

@ -45,7 +45,7 @@ public:
const enum ObKVCachePolicy policy = LRU);
protected:
virtual bool add_handle_ref(MBWrapper *mb_wrapper) = 0;
virtual void de_handle_ref(MBWrapper *mb_wrapper) = 0;
virtual void de_handle_ref(MBWrapper *mb_wrapper, const bool do_retire = true) = 0;
virtual int alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy,
const int64_t block_size, MBWrapper *&mb_wrapper) = 0;
virtual int free(MBWrapper *mb_wrapper) = 0;
@ -84,12 +84,12 @@ public:
ObKVMemBlockHandle *&mb_handle);
virtual int alloc_mbhandle(ObKVCacheInst &inst, ObKVMemBlockHandle *&mb_handle);
virtual int alloc_mbhandle(const ObKVCacheInstKey &inst_key, ObKVMemBlockHandle *&mb_handle);
virtual int free_mbhandle(ObKVMemBlockHandle *mb_handle);
virtual int free_mbhandle(ObKVMemBlockHandle *mb_handle, const bool do_retire);
virtual int mark_washable(ObKVMemBlockHandle *mb_handle);
virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle, const uint32_t seq_num);
virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle);
virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle);
virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true);
int64_t get_handle_ref_cnt(const ObKVMemBlockHandle *mb_handle);
virtual int64_t get_block_size() const { return block_size_; }
// implement functions of ObIMBWrapperMgr
@ -111,6 +111,7 @@ private:
const int64_t size_need_washed = INT64_MAX,
const bool force_flush = false);
int inner_push_memblock_info(const ObKVMemBlockHandle &handle, ObIArray<ObKVCacheStoreMemblockInfo> &memblock_infos);
void purge_mb_handle_retire_station();
private:
static const int64_t SYNC_WASH_MB_TIMEOUT_US = 100 * 1000; // 100ms
@ -213,9 +214,9 @@ private:
}
int insert_mb_handle(common::ObDLink *head, ObKVMemBlockHandle *mb_handle);
int remove_mb_handle(ObKVMemBlockHandle *mb_handle);
void retire_mb_handle(ObKVMemBlockHandle *mb_handle);
void retire_mb_handles(HazardList &retire_list);
int remove_mb_handle(ObKVMemBlockHandle *mb_handle, const bool do_retire);
void retire_mb_handle(ObKVMemBlockHandle *mb_handle, const bool do_retire);
void retire_mb_handles(HazardList &retire_list, const bool do_retire);
void reuse_mb_handles(HazardList &reclaim_list);
bool try_supply_mb(const int64_t mb_count);
private:

4
src/share/cache/ob_kvcache_struct.h vendored
View File

@ -261,12 +261,12 @@ public:
ObKVMemBlockHandle *&mb_handle) = 0;
virtual int alloc_mbhandle(ObKVCacheInst &inst, ObKVMemBlockHandle *&mb_handle) = 0;
virtual int alloc_mbhandle(const ObKVCacheInstKey &inst_key, ObKVMemBlockHandle *&mb_handle) = 0;
virtual int free_mbhandle(ObKVMemBlockHandle *mb_handle) = 0;
virtual int free_mbhandle(ObKVMemBlockHandle *mb_handle, const bool do_retire) = 0;
virtual int mark_washable(ObKVMemBlockHandle *mb_handle) = 0;
virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle, const uint32_t seq_num) = 0;
virtual bool add_handle_ref(ObKVMemBlockHandle *mb_handle) = 0;
virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle) = 0;
virtual void de_handle_ref(ObKVMemBlockHandle *mb_handle, const bool do_retire = true) = 0;
virtual int64_t get_block_size() const = 0;
};

6
src/share/cache/ob_working_set_mgr.cpp vendored
View File

@ -142,10 +142,10 @@ bool ObWorkingSet::add_handle_ref(WorkingSetMB *ws_mb)
return added;
}
void ObWorkingSet::de_handle_ref(WorkingSetMB *ws_mb)
void ObWorkingSet::de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire)
{
if (NULL != ws_mb) {
mb_handle_allocator_->de_handle_ref(ws_mb->mb_handle_);
mb_handle_allocator_->de_handle_ref(ws_mb->mb_handle_, do_retire);
}
}
@ -347,7 +347,7 @@ bool ObWorkingSet::try_reuse_mb(WorkingSetMB *ws_mb, ObKVMemBlockHandle *&mb_han
if (ATOMIC_BCAS((uint32_t*)(&reused_handle->status_), FULL, FREE)) {
if (reused_handle->handle_ref_.try_check_and_inc_seq_num(seq_num)) {
int ret = OB_SUCCESS;
if (OB_FAIL(mb_handle_allocator_->free_mbhandle(reused_handle))) {
if (OB_FAIL(mb_handle_allocator_->free_mbhandle(reused_handle, true /* do_retire */))) {
LOG_WARN("free_mbhandle failed", K(ret));
} else {
reused = true;

2
src/share/cache/ob_working_set_mgr.h vendored
View File

@ -74,7 +74,7 @@ public:
// implemnt functions of ObIKVCacheStore<WorkingSetMB>
virtual bool add_handle_ref(WorkingSetMB *ws_mb);
virtual void de_handle_ref(WorkingSetMB *ws_mb);
virtual void de_handle_ref(WorkingSetMB *ws_mb, const bool do_retire);
virtual int alloc(ObKVCacheInst &inst, const enum ObKVCachePolicy policy,
const int64_t block_size, WorkingSetMB *&ws_mb);
virtual int free(WorkingSetMB *ws_mb);