hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity
Files
883414e5c3cb938a9e278e325d53687ce3d4892c
oceanbase/src/share/detect/ob_detect_manager.cpp

829 lines
31 KiB
C++
Raw Blame History

/**
* Copyright (c) 2023 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "share/detect/ob_detect_manager.h"
#include "lib/ob_running_mode.h"
#include "share/rc/ob_context.h"
#include "lib/lock/ob_spin_lock.h"
namespace oceanbase {
namespace common {
ObDetectableIdGen::ObDetectableIdGen()
{
detect_sequence_id_ = ObTimeUtil::current_time();
callback_node_sequence_id_ = ObTimeUtil::current_time();
}
ObDetectableIdGen &ObDetectableIdGen::instance()
{
static ObDetectableIdGen d_id_gen;
return d_id_gen;
}
int ObDetectableIdGen::generate_detectable_id(ObDetectableId &detectable_id, uint64_t tenant_id)
{
int ret = OB_SUCCESS;
// use server id to ensure that the detectable_id is unique in cluster
uint64_t server_id = GCTX.server_id_;
if (!is_valid_server_id(server_id)) {
ret = OB_SERVER_IS_INIT;
LIB_LOG(WARN, "[DM] server id is invalid");
} else {
detectable_id.first_ = get_detect_sequence_id();
// use timestamp to ensure that the detectable_id is unique during the process
uint64_t timestamp = ObTimeUtility::current_time();
// [ server_id (16bits) ][ timestamp (32bits) ]
// only if qps > 2^48 or same server reboots after 2^48 the detectable_id be repeated
detectable_id.second_ = (server_id) << 48 | (timestamp & 0x0000FFFFFFFFFFFF);
detectable_id.tenant_id_ = tenant_id;
}
return ret;
}
int ObDMMultiDlist::init(int64_t tenant_id, int64_t bucket_num)
{
int ret = OB_SUCCESS;
bucket_num_ = bucket_num;
allocator_.set_attr(ObMemAttr(tenant_id, "ObDMMDL"));
locks_.set_allocator(&allocator_);
buckets_.set_allocator(&allocator_);
if (OB_FAIL(locks_.init(bucket_num_))) {
LIB_LOG(WARN, "[DM] failed to init locks_");
} else if (OB_FAIL(buckets_.init(bucket_num_))) {
LIB_LOG(WARN, "[DM] failed to init buckets_");
}
for (int64_t i = 0; OB_SUCC(ret) && i < bucket_num_; ++i) {
ObLockWrapper *lock = OB_NEWx(ObLockWrapper, &allocator_);
ObDList<ObDetectableIdDNode> *dlist = OB_NEWx(ObDList<ObDetectableIdDNode>, &allocator_);
if (OB_ISNULL(lock) || OB_ISNULL(dlist)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] failed to allocate");
} else if (OB_FAIL(locks_.push_back(lock))) {
LIB_LOG(WARN, "[DM] failed to push_back lock");
} else if (OB_FAIL(buckets_.push_back(dlist))) {
LIB_LOG(WARN, "[DM] failed to push_back dlist");
}
}
return ret;
}
void ObDMMultiDlist::destroy() {
for (int64_t i = 0; i < locks_.count(); ++i) {
locks_.at(i)->~ObLockWrapper();
allocator_.free(locks_.at(i));
}
for (int64_t i = 0; i < buckets_.count(); ++i) {
buckets_.at(i)->~ObDList<ObDetectableIdDNode>();
allocator_.free(buckets_.at(i));
}
locks_.destroy();
buckets_.destroy();
LIB_LOG(INFO, "[DM] destroy ObDMMultiDlist");
}
void ObDMMultiDlist::add_list_node_tail(const ObDetectableId &detectable_id, ObDetectableIdDNode *node)
{
int64_t bucket_id = detectable_id.first_ % bucket_num_;
{
ObLockGuard<common::ObSpinLock> lock_guard(*locks_.at(bucket_id));
buckets_.at(bucket_id)->add_last(node);
}
}
void ObDMMultiDlist::remove_list_node(const ObDetectableId &detectable_id, ObDetectableIdDNode *node)
{
int64_t bucket_id = detectable_id.first_ % bucket_num_;
{
ObLockGuard<common::ObSpinLock> lock_guard(*locks_.at(bucket_id));
buckets_.at(bucket_id)->remove(node);
}
}
void ObDMMultiDlist::pop_active_node(
hash::ObHashSet<ObDetectableId, hash::NoPthreadDefendMode> &still_need_check_id)
{
int ret = OB_SUCCESS;
int64_t cur_time = ObTimeUtil::current_time();
for (int64_t i = 0; OB_SUCC(ret) && i < bucket_num_; ++i) {
bool end_bucket_traverse = false;
// pop MAX_LOCK_LENGTH nodes per lock time
while (!end_bucket_traverse && OB_SUCC(ret)) {
int64_t poped_cnt = 0;
ObLockGuard<common::ObSpinLock> lock_guard(*locks_.at(i));
if (buckets_.at(i)->is_empty()) {
break;
}
ObDetectableIdDNode *cur = buckets_.at(i)->get_first();
ObDetectableIdDNode *next = nullptr;
while (OB_SUCC(ret) && ++poped_cnt <= MAX_LOCK_LENGTH) {
if (cur->activate_tm_ > cur_time) {
// all remain nodes in this bucket are not active, stopping traverse this bucket.
end_bucket_traverse = true;
break;
} else {
if (OB_FAIL(still_need_check_id.set_refactored(cur->detectable_id_))) {
if (OB_HASH_EXIST != ret) {
end_bucket_traverse = true;
LIB_LOG(WARN, "[DM] failed to set_refactored", K(ret), K(cur->detectable_id_));
break;
} else {
ret = OB_SUCCESS;
}
}
// pop cur node
next = cur->get_next();
buckets_.at(i)->remove(cur);
cur = next;
if (buckets_.at(i)->is_empty()) {
end_bucket_traverse = true;
break;
}
}
}
}
}
}
int ObDetectManager::mtl_new(ObDetectManager *&dm)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = MTL_ID();
dm = OB_NEW(ObDetectManager, ObMemAttr(tenant_id, "DetectManager"), tenant_id);
if (OB_ISNULL(dm)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] failed to alloc detect manager", K(ret));
}
return ret;
}
int ObDetectManager::mtl_init(ObDetectManager *&dm)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = MTL_ID();
int64_t mem_limit = lib::get_tenant_memory_limit(tenant_id);
double mem_factor = mem_limit >= MAX_TENANT_MEM_LIMIT ? 1.0 : (static_cast<double>(mem_limit) / MAX_TENANT_MEM_LIMIT);
// less memory for meta tenant
if (is_meta_tenant(tenant_id)) {
mem_factor = mem_factor * 0.01;
}
if (OB_FAIL(dm->init(GCTX.self_addr(), mem_factor))) {
LIB_LOG(WARN, "[DM] failed to init detect manager", K(ret));
}
return ret;
}
void ObDetectManager::mtl_destroy(ObDetectManager *&dm)
{
if (nullptr != dm) {
dm->destroy();
ob_delete(dm);
dm = nullptr;
}
}
void ObDetectManager::destroy()
{
int ret = OB_SUCCESS;
// destroy dm_multi_list_
dm_multi_list_.destroy();
// destroy node and callback in all_check_items_
FOREACH(iter, all_check_items_) {
const ObDetectableId &detectable_id = iter->first;
ObDetectCallbackNode *node = iter->second;
while (OB_NOT_NULL(node)) {
ObDetectCallbackNode *next_node = node->next_;
// DM is destroying means that all tenant threads have already exited,
// the remain detect callbacks must be
// ObSingleDfoDetectCB, ObTempTableDetectCB, or ObP2PDataHubDetectCB
int temp_ret = node->cb_->do_callback();
LIB_LOG(WARN, "[DM] do callback during mtl destroy",
K(temp_ret), K(node->cb_->get_trace_id()),
K(detectable_id), K(node->cb_->get_detect_callback_type()),
K(node->sequence_id_));
delete_cb_node(node);
node = next_node;
}
}
all_check_items_.destroy();
still_need_check_id_.destroy();
detectable_ids_.destroy();
DESTROY_CONTEXT(mem_context_);
LIB_LOG(INFO, "[DM] destory dm", K_(tenant_id));
}
int ObDetectManager::init(const ObAddr &self, double mem_factor)
{
int ret = OB_SUCCESS;
lib::ContextParam param;
param.set_mem_attr(tenant_id_, "DetectManager")
.set_properties(lib::ALLOC_THREAD_SAFE)
.set_parallel(4);
int64_t check_map_bucket_count = static_cast<int64_t>(DEFAULT_CHECK_MAP_BUCKETS_COUNT * mem_factor);
int64_t detectable_ids_bucket_count = static_cast<int64_t>(DEFAULT_CHECK_MAP_BUCKETS_COUNT * mem_factor);
int64_t still_need_check_id_bucket_count = static_cast<int64_t>(MIDDLE_SET_BUCKETS_COUNT * mem_factor);
if (IS_INIT) {
ret = OB_INIT_TWICE;
} else if (!self.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "[DM] invalid arguments", K(ret), K(self));
} else if (OB_FAIL(ROOT_CONTEXT->CREATE_CONTEXT(mem_context_, param))) {
LIB_LOG(WARN, "[DM] create memory context failed", K(ret));
} else if (OB_FAIL(all_check_items_.create(check_map_bucket_count,
"HashBuckDmChe",
"HashNodeDmChe",
tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] create hash table failed", K(ret));
} else if (OB_FAIL(detectable_ids_.create(detectable_ids_bucket_count,
"HashBuckDmId",
"HashNodeDmId",
tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] create hash set failed", K(ret));
} else if (OB_FAIL(still_need_check_id_.create(still_need_check_id_bucket_count,
"HashBuckDmId",
"HashNodeDmId",
tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] create hash set failed", K(ret));
} else if (OB_FAIL(dm_multi_list_.init(tenant_id_,
is_meta_tenant(tenant_id_) ? DM_MDLIST_BUCKETS_COUNT_FOR_META_TENANT : DM_MDLIST_BUCKETS_COUNT))) {
LIB_LOG(WARN, "[DM] failed to init ObDMMultiDlist", K(ret));
} else {
self_ = self;
is_inited_ = true;
LIB_LOG(INFO, "[DM] ObDetectManager init success", K(self), K_(tenant_id), K(mem_factor));
}
return ret;
}
int ObDetectManager::register_detectable_id(const ObDetectableId &detectable_id)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LIB_LOG(ERROR, "[DM] detect manager not inited", K(ret));
} else if (detectable_id.is_invalid()) {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "[DM] invaild detectable_id, please generate from ObDetectManager", K(ret), K(common::lbt()));
} else if (OB_FAIL(detectable_ids_.set_refactored(detectable_id, 0/* flag */))) {
if (OB_HASH_EXIST != ret) {
LIB_LOG(WARN, "[DM] detectable_ids_ set_refactored failed", K(ret));
} else {
LIB_LOG(WARN, "[DM] detectable_id already exists in detectable_ids_");
}
}
return ret;
}
int ObDetectManager::unregister_detectable_id(const ObDetectableId &detectable_id)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LIB_LOG(ERROR, "[DM] detect manager not inited", K(ret));
} else if (detectable_id.is_invalid()) {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "[DM] invaild detectable_id", K(common::lbt()));
} else if (OB_FAIL(detectable_ids_.erase_refactored((detectable_id)))) {
LIB_LOG(WARN, "[DM] detectable_ids_ erase_refactored failed", K(ret), K(detectable_id));
}
return ret;
}
int ObDetectManager::do_register_check_item(const ObDetectableId &detectable_id, ObDetectCallbackNode *cb_node,
uint64_t &node_sequence_id, bool need_ref)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(cb_node)) {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "[DM] invaild cb node pointer");
} else {
// if need_ref is true, which means that work thread may use cb, so add ref_count in case of deleting cb during ObDetectCallbackNodeExecuteCall
// typical scene: qc may change sqc's task state in callback upon receiving the report message from sqc, but dm may free callback already
if (need_ref) {
cb_node->cb_->inc_ref_count();
}
node_sequence_id = cb_node->sequence_id_;
// A slightly more complicated but safe inspection operation
// Since map does not provide the operation of "create or modify", nor does it provide the ability to hold bucket locks
// Therefore, add cyclic verification to prevent the occurrence of
// thread a failed to create --> thread b erased --> thread a failed to modify
// Such a situation
ObObDetectCallbackNodeSetCall set_node_call(this);
do {
if (OB_HASH_EXIST == (ret = all_check_items_.set_refactored(detectable_id, cb_node, 0, 0, 0, &set_node_call))) {
ObDetectCallbackNodeAddCall add_node_call(cb_node, this);
ret = all_check_items_.atomic_refactored(detectable_id, add_node_call);
// If it is an empty queue, it means that another thread wants to delete the node but unexpectedly got the lock by this thread
// So do not delete, try to put again according to HASH_NOT_EXIST
if (add_node_call.is_empty()) {
ret = OB_HASH_NOT_EXIST;
}
}
} while (ret == OB_HASH_NOT_EXIST);
// hashmap may set_refactored for alloc failed
if OB_FAIL(ret) {
delete_cb_node(cb_node);
}
}
if (OB_SUCC(ret)) {
LIB_LOG(DEBUG, "[DM] register_check_item", K(ret), K(detectable_id),
K(cb_node->cb_->get_detect_callback_type()), K(node_sequence_id));
}
return ret;
}
int ObDetectManager::unregister_check_item(const ObDetectableId &detectable_id, const uint64_t &node_sequence_id)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LIB_LOG(ERROR, "[DM] detect manager not inited");
} else if (detectable_id.is_invalid()) {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "[DM] invaild detectable_id", K(node_sequence_id) ,K(common::lbt()));
} else {
// CHECK_MAP key: ObDetectableId, CHECK_MAP value: linked list with node type ObDetectCallbackNode*
// remove node from CHECK_MAP with detectable_id(CHECK_MAP key) and node_sequence_id(identify specific node in the linked list)
// remove kv pair from CHECK_MAP if this node is the last one in the linked list
ObDetectCallbackNodeRemoveCall remove_node_call(this, node_sequence_id);
bool is_erased = false;
if (OB_FAIL(all_check_items_.erase_if(detectable_id, remove_node_call, is_erased))) {
if (OB_HASH_NOT_EXIST == ret) {
// if not found, the possible reason is that node is removed by ObDetectCallbackNodeExecuteCall
LIB_LOG(TRACE, "[DM] unregister cb failed, maybe removed by other thread",
K(detectable_id), K(node_sequence_id));
} else {
LIB_LOG(WARN, "[DM] unregister cb failed", K(ret), K(detectable_id), K(node_sequence_id));
}
}
}
LIB_LOG(DEBUG, "[DM] unregister_check_item", K(ret), K(detectable_id), K(node_sequence_id));
return ret;
}
int ObDetectManager::gather_requests(REQUEST_MAP &req_map, lib::MemoryContext &req_map_context)
{
int ret = OB_SUCCESS;
IGNORE_RETURN dm_multi_list_.pop_active_node(still_need_check_id_);
ObSEArray<ObDetectableId, 32> remove_list;
FOREACH(iter, still_need_check_id_) {
const ObDetectableId &detectable_id = iter->first;
ObDetectReqGetCall req_get_call(req_map, req_map_context);
ret = all_check_items_.read_atomic(detectable_id, req_get_call);
if (OB_HASH_NOT_EXIST == ret) {
// already unregistered, remove it. if push_back failed, let next detect loop do this
if (OB_FAIL(remove_list.push_back(detectable_id))) {
LIB_LOG(WARN, "[DM] failed to push_back to remove_list", K(ret));
}
ret = OB_SUCCESS;
}
}
ARRAY_FOREACH_NORET(remove_list, idx) {
if (OB_FAIL(still_need_check_id_.erase_refactored(remove_list.at(idx)))) {
LIB_LOG(WARN, "[DM] failed to erase_refactored from still_need_check_id_");
}
}
LIB_LOG(DEBUG, "[DM] gather_requests ", K(req_map.size()), K(ret));
return ret;
}
void ObDetectManager::do_detect_local(const ObDetectableId &detectable_id)
{
int ret = OB_SUCCESS;
bool task_alive = is_task_alive(detectable_id);
if (!task_alive) {
ObCheckStateFinishCall check_state_finish_call(self_);
if (OB_SUCC(all_check_items_.read_atomic(detectable_id, check_state_finish_call))) {
if (check_state_finish_call.is_finished()) {
// nop
} else {
// peer exits unexpectly, do detect callbacks
ObDetectCallbackNodeExecuteCall execute_call(this, detectable_id, self_);
bool is_erased = false;
IGNORE_RETURN all_check_items_.erase_if(detectable_id, execute_call, is_erased);
}
}
}
}
void ObDetectManager::do_handle_one_result(const ObDetectableId &detectable_id, const obrpc::ObDetectRpcStatus &rpc_status)
{
int ret = OB_SUCCESS;
ObCheckStateFinishCall check_state_finish_call(rpc_status.dst_);
if (OB_SUCC(all_check_items_.read_atomic(detectable_id, check_state_finish_call))) {
if (check_state_finish_call.is_finished()) {
// nop
} else {
ObDetectCallbackNodeExecuteCall execute_call(this, detectable_id, rpc_status.dst_);
bool is_erased = false;
IGNORE_RETURN all_check_items_.erase_if(detectable_id, execute_call, is_erased);
}
}
}
void ObDetectManager::delete_cb_node(ObDetectCallbackNode *&cb_node)
{
int ret = OB_SUCCESS;
// cb and cb_node allocated simultaneously, just need to free cb_node
LIB_LOG(DEBUG, "[DM] dm free cb ", K(cb_node->cb_));
cb_node->cb_->destroy();
cb_node->cb_ = nullptr;
LIB_LOG(DEBUG, "[DM] dm free cbnode ", K(cb_node));
mem_context_->free(cb_node);
cb_node = nullptr;
}
int ObDetectManager::ObObDetectCallbackNodeSetCall::operator()(const hash::HashMapPair<ObDetectableId,
ObDetectCallbackNode *> &entry)
{
int ret = OB_SUCCESS;
(void) dm_->dm_multi_list_.add_list_node_tail(entry.first, &entry.second->cb_->d_node_);
return ret;
}
void ObDetectManager::ObDetectCallbackNodeAddCall::operator()(hash::HashMapPair<ObDetectableId,
ObDetectCallbackNode *> &entry)
{
// The map only stores the head of the linked list, so add it directly from the head
if (entry.second != nullptr) {
cb_node_->next_ = entry.second;
entry.second->prev_ = cb_node_;
entry.second = cb_node_;
(void) dm_->dm_multi_list_.add_list_node_tail(entry.first, &cb_node_->cb_->d_node_);
is_empty_ = false;
} else {
is_empty_ = true;
}
}
bool ObDetectManager::ObDetectCallbackNodeRemoveCall::operator()(hash::HashMapPair<ObDetectableId,
ObDetectCallbackNode *> &entry)
{
auto node = entry.second;
int node_cnt = 0; // node_cnt in linked list, only if node_cnt == 0 the kv pair can be deleted.
while (OB_NOT_NULL(node)) {
node_cnt++;
if (node_sequence_id_ == node->sequence_id_) {
found_node_ = true;
if (node->next_ != nullptr) {
node->next_->prev_ = node->prev_;
// loop will be break after found node, but next node is not null
// add node_cnt to prevent to delete the kv pair
node_cnt++;
}
if (node->prev_ != nullptr) {
node->prev_->next_ = node->next_;
} else {
// Prev is empty, which means that the current deleted element is the head of the linked list pointed to by map_value, and head is set to next
entry.second = node->next_;
}
(void) dm_->dm_multi_list_.remove_list_node(entry.first, &node->cb_->d_node_);
dm_->delete_cb_node(node);
node_cnt--;
break;
}
node = node->next_;
}
// 0 == node_cnt means that the linked list is empty, return true so that erase_if can erase the kv pair
return (0 == node_cnt);
}
bool ObDetectManager::ObDetectCallbackNodeExecuteCall::operator()(hash::HashMapPair<ObDetectableId,
ObDetectCallbackNode *> &entry)
{
int ret = OB_SUCCESS;
const ObDetectableId &detectable_id = entry.first;
ObDetectCallbackNode *node = entry.second;
int node_cnt = 0; // node_cnt in linked list, remove kv pair if the last node is removed
while (OB_NOT_NULL(node)) {
ret = OB_SUCCESS;
node_cnt++;
ObDetectCallbackNode *next_node = node->next_;
// if a callback can be reentrant, don't set is_executed so that it can be do for several times
// typical scene: qc detects sqc, if at least 2 sqc failed, both of them should be set not_need_report,
// the callback should be reentrant and node not set to executed
if (!node->is_executed()) {
LIB_LOG(WARN, "[DM] DM found peer not exist, execute detect callback",
K(node->cb_->get_trace_id()), K(from_svr_addr_),
K(detectable_id), K(node->cb_->get_detect_callback_type()), K(node->sequence_id_));
node->cb_->set_from_svr_addr(from_svr_addr_);
if (OB_FAIL(node->cb_->do_callback())) {
// if do_callback failed, reset state to running for next detect loop
node->cb_->atomic_set_running(from_svr_addr_);
LIB_LOG(WARN, "[DM] failed to do_callback",
K(node->cb_->get_trace_id()), K(from_svr_addr_), K(detectable_id),
K(node->cb_->get_detect_callback_type()), K(node->sequence_id_));
} else {
if (!node->cb_->reentrant()) {
node->set_executed();
}
// ref_count > 0 means that cb is still referred by work thread, don‘t remove it from the linked list
int64_t ref_count = node->cb_->get_ref_count();
if (0 == ref_count) {
if (node->next_ != nullptr) {
node->next_->prev_ = node->prev_;
}
if (node->prev_ != nullptr) {
node->prev_->next_ = node->next_;
} else {
// Prev is empty, which means that the current deleted element is the head of the linked list pointed to by map_value, and head is set to next
entry.second = node->next_;
}
(void) dm_->dm_multi_list_.remove_list_node(entry.first, &node->cb_->d_node_);
dm_->delete_cb_node(node);
node_cnt--;
}
}
}
node = next_node;
}
return 0 == node_cnt;
}
void ObDetectManager::ObCheckStateFinishCall::operator()(hash::HashMapPair<ObDetectableId,
ObDetectCallbackNode *> &entry)
{
ObDetectCallbackNode *node = entry.second;
// check task has already been marked as finished
while (OB_NOT_NULL(node)) {
ObTaskState state = ObTaskState::RUNNING;
if (OB_SUCCESS == node->cb_->atomic_set_finished(addr_, &state)) {
if (ObTaskState::FINISHED == state) {
finished_ = true;
}
break;
}
node = node->next_;
}
}
void ObDetectManager::ObDetectReqGetCall::operator()(hash::HashMapPair<ObDetectableId, ObDetectCallbackNode *> &entry)
{
int ret = OB_SUCCESS;
const ObDetectableId &peer_id = entry.first;
ObDetectCallbackNode *cb_node = entry.second;
while (OB_NOT_NULL(cb_node)) {
if (!cb_node->is_executed()) {
ObIArray<ObPeerTaskState> &peer_states = cb_node->cb_->get_peer_states();
ARRAY_FOREACH_NORET(peer_states, idx) {
ObPeerTaskState &peer_state = peer_states.at(idx);
// only detect running tasks
if ((int32_t)ObTaskState::FINISHED == ATOMIC_LOAD((int32_t*)&peer_state)) {
continue;
}
obrpc::ObTaskStateDetectReq **req_ptr_ptr = req_map_.get(peer_state.peer_addr_);
if (OB_NOT_NULL(req_ptr_ptr)) {
if (OB_FAIL((*req_ptr_ptr)->peer_ids_.set_refactored(peer_id, 0/* flag */))) {
if (OB_HASH_EXIST != ret) {
LIB_LOG(WARN, "[DM] peer_ids_ set_refactored failed", K(peer_id),
K(peer_state.peer_addr_), K(cb_node->cb_->get_trace_id()));
} else {
// this peer id has already been set, ignore
ret = OB_SUCCESS;
}
}
} else {
obrpc::ObTaskStateDetectReq *req_ptr =
OB_NEWx(obrpc::ObTaskStateDetectReq, &req_map_context_->get_arena_allocator());
if (OB_ISNULL(req_ptr)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] failed to new ObTaskStateDetectReq", K(peer_id),
K(peer_state.peer_addr_), K(cb_node->cb_->get_trace_id()));
} else if (OB_FAIL(req_ptr->peer_ids_.set_refactored(peer_id, 0/* flag */))) {
LIB_LOG(WARN, "[DM] peer_ids_ set_refactored failed", K(peer_id),
K(peer_state.peer_addr_), K(cb_node->cb_->get_trace_id()));
} else if (OB_FAIL(req_map_.set_refactored(peer_state.peer_addr_, req_ptr))) {
LIB_LOG(WARN, "[DM] req_map_ set req failed", K(peer_state.peer_addr_),
K(ret), K(peer_state.peer_addr_), K(cb_node->cb_->get_trace_id()));
}
}
}
}
cb_node = cb_node->next_;
}
}
ObDetectManagerThread &ObDetectManagerThread::instance()
{
static ObDetectManagerThread dm_thread;
return dm_thread;
}
int ObDetectManagerThread::init(const ObAddr &self, rpc::frame::ObReqTransport *transport)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
} else if (!self.is_valid() || nullptr == transport) {
ret = OB_INVALID_ARGUMENT;
LIB_LOG(WARN, "[DM] invalid arguments", K(ret), K(self), KP(transport));
} else if (OB_FAIL(rpc_proxy_.init(transport, self))) {
LIB_LOG(WARN, "[DM] rpc_proxy_.init failed", K(ret));
} else if (OB_FAIL(req_map_.create(DEFAULT_REQUEST_MAP_BUCKETS_COUNT,
"HashBuckDmReq",
"HashNodeDmReq"))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] create hash set failed", K(ret));
} else if (OB_FAIL(cond_.init(common::ObWaitEventIds::DEFAULT_COND_WAIT))) {
LIB_LOG(WARN, "[DM] failed to init cond_", K(ret));
} else if (OB_FAIL(TG_SET_RUNNABLE_AND_START(lib::TGDefIDs::DetectManager, *this))) {
LIB_LOG(WARN, "[DM] schedule detect without timer failed", K(ret));
} else {
self_ = self;
is_inited_ = true;
LIB_LOG(INFO, "[DM] ObDetectManagerThread init success", K(self));
}
return ret;
}
void ObDetectManagerThread::run1()
{
lib::set_thread_name("ObDetectManagerThread");
IGNORE_RETURN detect();
}
int ObDetectManagerThread::detect() {
static int loop_time = 0;
int ret = OB_SUCCESS;
while (!has_set_stop()) {
ret = OB_SUCCESS;
const int64_t start_time = ObTimeUtility::current_time();
int64_t send_cnt = 0; // mark rpc cnt
// 1. Gather all request from CHECK_MAP into REQUEST_MAP, clustered by addr.
// 2. Send all requests ip by ip. If is local, directly access detectable_ids_.
// 3. Process rpc result
// All requests are allocated from TEMP_CONTEXT to simplify memory management.
CREATE_WITH_TEMP_CONTEXT(lib::ContextParam().set_label(ObModIds::OB_TEMP_VARIABLES)) {
lib::MemoryContext &temp_mem_context = CURRENT_CONTEXT;
common::ObArray<uint64_t> tenant_ids;
if (OB_ISNULL(GCTX.omt_)) {
ret = OB_ERR_UNEXPECTED;
LIB_LOG(WARN, "[DM] unexpected null of GCTX.omt_", K(ret));
} else if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids))) {
LIB_LOG(WARN, "[DM] fail to get_mtl_tenant_ids", K(ret));
} else {
for (int64_t i = 0; i < tenant_ids.size(); i++) {
MTL_SWITCH(tenant_ids.at(i)) {
ObDetectManager* dm = MTL(ObDetectManager*);
if (OB_ISNULL(dm)) {
LIB_LOG(WARN, "[DM] dm is null", K(tenant_ids.at(i)));
} else if (OB_FAIL(dm->gather_requests(req_map_, temp_mem_context))) {
LIB_LOG(WARN, "[DM] failed to gather_requests", K(tenant_ids.at(i)));
}
}
// ignore errors at switching tenant
ret = OB_SUCCESS;
}
if (FALSE_IT(send_requests(req_map_, send_cnt, temp_mem_context))) {
} else if (FALSE_IT(handle_rpc_results(send_cnt, temp_mem_context))) {
}
FOREACH(it, req_map_) {
it->second->destroy();
}
}
}
req_map_.reuse();
const int64_t cost_time = ObTimeUtility::current_time() - start_time;
if (cost_time > DETECT_COST_TIME_THRESHOLD) {
LIB_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "[DM] detect_loop_ cost too much time", K(cost_time));
}
int32_t sleep_time = DETECT_INTERVAL - static_cast<const int32_t>(cost_time);
if (sleep_time < 0) {
sleep_time = 0;
} else {
ob_usleep(sleep_time);
}
++loop_time;
LIB_LOG(DEBUG, "[DM] detect has execute ", K(loop_time));
}
return ret;
}
void ObDetectManagerThread::send_requests(REQUEST_MAP &req_map, int64_t &send_cnt, lib::MemoryContext &mem_context)
{
FOREACH(iter, req_map) {
const common::ObAddr &dst = iter->first;
obrpc::ObTaskStateDetectReq *req = iter->second;
if (dst == self_) {
detect_local(req);
} else {
detect_remote(dst, req, send_cnt, mem_context);
}
}
}
void ObDetectManagerThread::detect_local(const obrpc::ObTaskStateDetectReq *req)
{
int ret = OB_SUCCESS;
FOREACH(iter, req->peer_ids_) {
const ObDetectableId &detectable_id = iter->first;
MTL_SWITCH(detectable_id.tenant_id_) {
ObDetectManager* dm = MTL(ObDetectManager*);
if (OB_ISNULL(dm)) {
LIB_LOG(WARN, "[DM] dm is null", K(detectable_id.tenant_id_));
} else {
dm->do_detect_local(detectable_id);
}
}
}
}
void ObDetectManagerThread::detect_remote(const common::ObAddr &dst,
const obrpc::ObTaskStateDetectReq *req, int64_t &send_cnt, lib::MemoryContext &mem_context)
{
int ret = OB_SUCCESS;
obrpc::ObDetectRpcStatus *rpc_status = OB_NEWx(obrpc::ObDetectRpcStatus, &mem_context->get_malloc_allocator(), dst);
LIB_LOG(DEBUG, "[DM] dm new rpc_status");
if (OB_ISNULL(rpc_status)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LIB_LOG(WARN, "[DM] failed to allocate rpc_status");
} else if (OB_FAIL(rpc_statuses_.push_back(rpc_status))) {
rpc_status->destroy();
mem_context->free(rpc_status);
LIB_LOG(WARN, "[DM] failed to push_back rpc_status into rpc_statuses_");
} else {
obrpc::ObTaskStateDetectAsyncCB cb(cond_, rpc_status, dst);
if (OB_FAIL(rpc_proxy_.to(dst).by(OB_SYS_TENANT_ID).timeout(DETECT_MSG_TIMEOUT).detect_task_state(*req, &cb))) {
rpc_statuses_.pop_back();
rpc_status->destroy();
mem_context->free(rpc_status);
LIB_LOG(WARN, "[DM] failed do rpc detect_task_state");
} else {
++send_cnt;
}
}
}
void ObDetectManagerThread::handle_rpc_results(int64_t &send_cnt, lib::MemoryContext &mem_context)
{
int ret = OB_SUCCESS;
int64_t return_cb_cnt = 0;
while (return_cb_cnt < send_cnt) {
ObThreadCondGuard guard(cond_);
// wait for timeout or until notified.
cond_.wait_us(COND_WAIT_TIME_USEC);
ARRAY_FOREACH_NORET(rpc_statuses_, idx) {
obrpc::ObDetectRpcStatus &rpc_status = *rpc_statuses_.at(idx);
if (!rpc_status.is_visited() && rpc_status.is_timeout()) {
return_cb_cnt++;
rpc_status.set_visited(true);
} else if (!rpc_status.is_visited() && rpc_status.is_processed()) {
return_cb_cnt++;
rpc_status.set_visited(true);
IGNORE_RETURN handle_one_result(rpc_status);
}
}
}
ARRAY_FOREACH_NORET(rpc_statuses_, idx) {
obrpc::ObDetectRpcStatus *rpc_status = rpc_statuses_.at(idx);
LIB_LOG(DEBUG, "[DM] dm free rpc_status");
rpc_status->destroy();
mem_context->get_malloc_allocator().free(rpc_status);
}
rpc_statuses_.reset();
}
void ObDetectManagerThread::handle_one_result(const obrpc::ObDetectRpcStatus &rpc_status)
{
int ret = OB_SUCCESS;
const obrpc::ObTaskStateDetectResp &cb_result = rpc_status.response_;
ARRAY_FOREACH_NORET(cb_result.task_infos_, idx) {
const obrpc::TaskInfo &task = cb_result.task_infos_.at(idx);
if (common::ObTaskState::FINISHED == task.task_state_) {
const ObDetectableId &detectable_id = task.task_id_;
MTL_SWITCH(detectable_id.tenant_id_) {
ObDetectManager* dm = MTL(ObDetectManager*);
if (OB_ISNULL(dm)) {
LIB_LOG(WARN, "[DM] dm is null", K(detectable_id.tenant_id_));
} else {
dm->do_handle_one_result(detectable_id, rpc_status);
}
}
}
}
}
} // end namespace common
} // end namespace oceanbase
Reference in New Issue View Git Blame Copy Permalink