hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[DeadLock.bad_case] fix remote sql retry may deadlock

Browse Source
This commit is contained in:
obdev
2023-02-15 14:17:33 +00:00
committed by ob-robot
parent efffd594cc
commit facae95a7d
11 changed files with 133 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/share/deadlock/ob_deadlock_detector_common_define.h
View File

@ -61,6 +61,7 @@ public:
// build a directed dependency relationship to other
virtual int block(const ObDependencyResource &) = 0;
virtual int block(const BlockCallBack &) = 0;
virtual int get_block_list(common::ObIArray<ObDependencyResource> &cur_list) const = 0;
// releace block list
virtual int replace_block_list(const common::ObIArray<ObDependencyResource> &) = 0;
// remove a directed dependency relationship to other

26
src/share/deadlock/ob_deadlock_detector_mgr.h
View File

@ -126,6 +126,8 @@ public:
template<typename T>
int replace_block_list(const T &src_key,
const common::ObIArray<ObDependencyResource> &new_list);
template<typename T>
int get_block_list(const T &src_key, common::ObIArray<ObDependencyResource> &cur_list);
// remove directed dependency relationship between two detector
template<typename T1, typename T2>
int activate(const T1 &src_key, const T2 &dest_key);
@ -450,6 +452,30 @@ int ObDeadLockDetectorMgr::replace_block_list(const T &src_key,
return ret;
#undef PRINT_WRAPPER
}
template<typename T>
int ObDeadLockDetectorMgr::get_block_list(const T &src_key,
common::ObIArray<ObDependencyResource> &cur_list)
{
CHECK_INIT();
CHECK_ENABLED();
#define PRINT_WRAPPER KR(ret), K(src_key), K(cur_list)
int ret = common::OB_SUCCESS;
DetectorRefGuard ref_guard;
UserBinaryKey src_user_key;
if (OB_FAIL(src_user_key.set_user_key(src_key))) {
DETECT_LOG(WARN, "src_key serialzation failed", PRINT_WRAPPER);
} else if (OB_FAIL(get_detector_(src_user_key, ref_guard))) {
DETECT_LOG(WARN, "get_detector failed", PRINT_WRAPPER);
} else if (OB_FAIL(ref_guard.get_detector()->get_block_list(cur_list))) {
DETECT_LOG(WARN, "get block list failed", PRINT_WRAPPER);
} else {
// DETECT_LOG(INFO, "replace block list success", PRINT_WRAPPER);
}
return ret;
#undef PRINT_WRAPPER
}
// call for removing directed dependency relationship between two detector(both in local)
// thread-safe guaranteed
//

27
src/share/deadlock/ob_lcl_scheme/ob_lcl_node.cpp
View File

@ -325,6 +325,19 @@ int ObLCLNode::activate_all()
#undef PRINT_WRAPPER
}
int ObLCLNode::get_block_list(common::ObIArray<ObDependencyResource> &cur_list) const
{
#define PRINT_WRAPPER KR(ret), K(*this)
int ret = OB_SUCCESS;
DETECT_TIME_GUARD(100_ms);
LockGuard lock_guard(lock_);
if (OB_FAIL(cur_list.assign(block_list_))) {
DETECT_LOG_(WARN, "fail to get block list", PRINT_WRAPPER);
}
return ret;
#undef PRINT_WRAPPER
}
int ObLCLNode::replace_block_list(const ObIArray<ObDependencyResource> &new_list)
{
#define PRINT_WRAPPER KR(ret), K(*this), K(new_list)
@ -528,15 +541,6 @@ int ObLCLNode::process_lcl_message(const ObLCLMessage &lcl_msg)
}
ret = OB_SUCCESS;
}
if (INVALID_VALUE != lclv_snapshot &&
public_label_snapshot.is_valid() &&
!blocklist_snapshot.empty()) {
CLICK();
(void)broadcast_(blocklist_snapshot, lclv_snapshot, public_label_snapshot);
DETECT_LOG_(TRACE, "receive lcl msg and keep broadcastiing", KR(ret), K(lcl_msg), K(*this));
}
if (detected_flag) {
CLICK();
ObDeadLockCollectInfoMessage collect_info_message;
@ -815,8 +819,7 @@ int ObLCLNode::push_state_to_downstreams_with_lock_()
DETECT_LOG_(WARN, "not waiting", K(*this), K_(last_report_waiting_for_period));
} else {
int64_t detector_id = private_label_.get_id();
DETECT_LOG_(INFO, "waiting for",
KP(this), K(detector_id), K_(self_key), K(blocklist_snapshot));
DETECT_LOG_(INFO, "waiting for", K_(self_key), K(blocklist_snapshot), KPC(this));
}
if (!parent_list_.empty()) {
@ -886,7 +889,7 @@ void ObLCLNode::PushStateTask::runTimerTask()
lcl_node_.timeout_ts_ + 10_min :
lcl_node_.created_time_ + 1_hour;
if (current_ts > warn_threshold_ts) {
DETECT_LOG(ERROR, "long lived lcl node, maybe leaked", K(*this));
DETECT_LOG(WARN, "long lived lcl node, maybe leaked", K(*this));
}
if (false == ATOMIC_LOAD(&lcl_node_.is_timer_task_canceled_)) {
if (expected_executed_ts > current_ts) {

5
src/share/deadlock/ob_lcl_scheme/ob_lcl_node.h
View File

@ -109,6 +109,7 @@ public:
// build a directed dependency relationship to other
int block(const ObDependencyResource &) override;
int block(const BlockCallBack &) override;
int get_block_list(common::ObIArray<ObDependencyResource> &cur_list) const override;
// releace block list
int replace_block_list(const common::ObIArray<ObDependencyResource> &) override;
// remove a directed dependency relationship to other
@ -119,9 +120,9 @@ public:
int process_collect_info_message(const ObDeadLockCollectInfoMessage &) override;
// handle message for scheme LCL
int process_lcl_message(const ObLCLMessage &) override;
TO_STRING_KV(KP(this), K_(self_key), K_(parent_key), K_(timeout_ts), K_(lclv), K_(private_label),
TO_STRING_KV(KP(this), K_(self_key), K_(parent_key), KTIME_(timeout_ts), K_(lclv), K_(private_label),
K_(public_label), K_(detect_callback),
K_(auto_activate_when_detected), K_(created_time), K_(allow_detect_time),
K_(auto_activate_when_detected), KTIME_(created_time), KTIME_(allow_detect_time),
K_(is_timer_task_canceled), K_(block_list), K_(parent_list),
K_(lcl_period), K_(last_send_collect_info_period), K(block_callback_list_.count()))
private: