diff --git a/deps/oblib/src/common/CMakeLists.txt b/deps/oblib/src/common/CMakeLists.txt index cbe8f9b562..adb692c4c3 100644 --- a/deps/oblib/src/common/CMakeLists.txt +++ b/deps/oblib/src/common/CMakeLists.txt @@ -59,4 +59,12 @@ ob_set_subtarget(oblib_common storage storage/ob_sequence.cpp ) +if (OB_ERRSIM) + ob_set_subtarget(oblib_common errsim_module + errsim_module/ob_errsim_module_type.cpp + errsim_module/ob_tenant_errsim_event.cpp + errsim_module/ob_errsim_module_interface.cpp + ) +endif() + ob_lib_add_target(oblib_common) diff --git a/deps/oblib/src/common/errsim_module/ob_errsim_module_interface.cpp b/deps/oblib/src/common/errsim_module/ob_errsim_module_interface.cpp new file mode 100644 index 0000000000..680e614a71 --- /dev/null +++ b/deps/oblib/src/common/errsim_module/ob_errsim_module_interface.cpp @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_errsim_module_interface.h" + +#include +#include +#include +#include "lib/ob_define.h" + +using namespace oceanbase::common; + +namespace oceanbase { +namespace common { + +int __attribute__((weak)) build_tenant_errsim_moulde( + const uint64_t tenant_id, + const int64_t config_version, + const common::ObArray> &module_array, + const int64_t percentage) +{ + int ret = OB_SUCCESS; + UNUSED(tenant_id); + UNUSED(config_version); + UNUSED(module_array); + UNUSED(percentage); + return ret; +} + +bool __attribute__((weak)) is_errsim_module( + const uint64_t tenant_id, + const ObErrsimModuleType::TYPE &type) +{ + bool b_ret = false; + UNUSED(tenant_id); + UNUSED(type); + return b_ret; +} + +int __attribute__((weak)) add_tenant_errsim_event( + const uint64_t tenant_id, + const ObTenantErrsimEvent &event) +{ + int ret = OB_SUCCESS; + UNUSED(tenant_id); + UNUSED(event); + return ret; +} + +} // common +} // oceanbase diff --git a/deps/oblib/src/common/errsim_module/ob_errsim_module_interface.h b/deps/oblib/src/common/errsim_module/ob_errsim_module_interface.h new file mode 100644 index 0000000000..57b15db709 --- /dev/null +++ b/deps/oblib/src/common/errsim_module/ob_errsim_module_interface.h @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_COMMON_ERRSIM_MODULE_OB_ERRSIM_MODULE_INTERFACE_H_ +#define OCEANBASE_COMMON_ERRSIM_MODULE_OB_ERRSIM_MODULE_INTERFACE_H_ + +#include +#include "lib/utility/ob_macro_utils.h" +#include "lib/utility/utility.h" +#include "ob_errsim_module_type.h" +#include "lib/container/ob_array.h" +#include "ob_tenant_errsim_event.h" + +namespace oceanbase +{ +namespace common +{ + +int build_tenant_errsim_moulde( + const uint64_t tenant_id, + const int64_t config_version, + const common::ObArray> &module_array, + const int64_t percentage); +bool is_errsim_module( + const uint64_t tenant_id, + const ObErrsimModuleType::TYPE &type); +int add_tenant_errsim_event( + const uint64_t tenant_id, + const ObTenantErrsimEvent &event); + +} // namespace common +} // namespace oceanbase + +#endif // OCEANBASE_COMMON_ERRSIM_MODULE_OB_ERRSIM_MODULE_INTERFACE_H_ diff --git a/deps/oblib/src/common/errsim_module/ob_errsim_module_type.cpp b/deps/oblib/src/common/errsim_module/ob_errsim_module_type.cpp new file mode 100644 index 0000000000..227741a699 --- /dev/null +++ b/deps/oblib/src/common/errsim_module/ob_errsim_module_type.cpp @@ -0,0 +1,106 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_errsim_module_type.h" + +namespace oceanbase +{ +using namespace lib; + +namespace common +{ + +static const char *OB_ERRSIM_MODULE_TYPES[] = { + "NONE", + "ALL", + "MIGRATION", + "TRANSFER", +}; + +void ObErrsimModuleType::reset() +{ + type_ = ObErrsimModuleType::ERRSIM_MODULE_MAX; +} + +bool ObErrsimModuleType::is_valid() const +{ + return ObErrsimModuleTypeHelper::is_valid(type_); +} + +const char *ObErrsimModuleType::get_str() +{ + return ObErrsimModuleTypeHelper::get_str(type_); +} + +bool ObErrsimModuleType::operator == (const ObErrsimModuleType &other) const +{ + bool is_same = true; + if (this == &other) { + // same + } else { + is_same = type_ == other.type_; + } + return is_same; +} + +int64_t ObErrsimModuleType::hash() const +{ + int64_t hash_value = 0; + hash_value = common::murmurhash( + &type_, sizeof(type_), hash_value); + return hash_value; +} + +int ObErrsimModuleType::hash(uint64_t &hash_val) const +{ + hash_val = hash(); + return OB_SUCCESS; +} + +OB_SERIALIZE_MEMBER(ObErrsimModuleType, type_); + +const char *ObErrsimModuleTypeHelper::get_str(const ObErrsimModuleType::TYPE &type) +{ + const char *str = nullptr; + if (!is_valid(type)) { + str = "UNKNOWN"; + } else { + str = OB_ERRSIM_MODULE_TYPES[type]; + } + return str; +} + +ObErrsimModuleType::TYPE ObErrsimModuleTypeHelper::get_type(const char *type_str) +{ + ObErrsimModuleType::TYPE type = ObErrsimModuleType::ERRSIM_MODULE_MAX; + + const int64_t count = ARRAYSIZEOF(OB_ERRSIM_MODULE_TYPES); + STATIC_ASSERT(static_cast(ObErrsimModuleType::ERRSIM_MODULE_MAX) == count, "type count mismatch"); + for (int64_t i = 0; i < count; ++i) { + if (0 == strcmp(type_str, OB_ERRSIM_MODULE_TYPES[i])) { + type = static_cast(i); + break; + } + } + return type; +} + +bool ObErrsimModuleTypeHelper::is_valid(const ObErrsimModuleType::TYPE &type) +{ + return type >= ObErrsimModuleType::ERRSIM_MODULE_NONE + && type < ObErrsimModuleType::ERRSIM_MODULE_MAX; +} + + +} //common +} //oceanbase diff --git a/deps/oblib/src/common/errsim_module/ob_errsim_module_type.h b/deps/oblib/src/common/errsim_module/ob_errsim_module_type.h new file mode 100644 index 0000000000..50d8f88f45 --- /dev/null +++ b/deps/oblib/src/common/errsim_module/ob_errsim_module_type.h @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_SHARE_ERRSIM_MODULE_OB_ERRSIM_MODULE_TYPE_H_ +#define SRC_SHARE_ERRSIM_MODULE_OB_ERRSIM_MODULE_TYPE_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" + +namespace oceanbase +{ +namespace common +{ + +struct ObErrsimModuleType final +{ + OB_UNIS_VERSION(1); +public: + enum TYPE + { + ERRSIM_MODULE_NONE = 0, + ERRSIM_MODULE_ALL = 1, + ERRSIM_MODULE_MIGRATION = 2, + ERRSIM_MODULE_TRANSFER = 3, + ERRSIM_MODULE_MAX + }; + ObErrsimModuleType() : type_(ERRSIM_MODULE_NONE) {} + explicit ObErrsimModuleType(const ObErrsimModuleType::TYPE &type) : type_(type) {} + ~ObErrsimModuleType() = default; + void reset(); + bool is_valid() const; + const char *get_str(); + bool operator == (const ObErrsimModuleType &other) const; + int hash(uint64_t &hash_val) const; + int64_t hash() const; + TO_STRING_KV(K_(type)); + TYPE type_; +}; + +struct ObErrsimModuleTypeHelper final +{ + static const char *get_str(const ObErrsimModuleType::TYPE &type); + static ObErrsimModuleType::TYPE get_type(const char *type_str); + static bool is_valid(const ObErrsimModuleType::TYPE &type); + static const int64_t MAX_TYPE_NAME_LENGTH = 16; +}; + + +} // namespace common +} // namespace oceanbase +#endif diff --git a/deps/oblib/src/common/errsim_module/ob_tenant_errsim_event.cpp b/deps/oblib/src/common/errsim_module/ob_tenant_errsim_event.cpp new file mode 100644 index 0000000000..fe9e71cce5 --- /dev/null +++ b/deps/oblib/src/common/errsim_module/ob_tenant_errsim_event.cpp @@ -0,0 +1,58 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_tenant_errsim_event.h" + +namespace oceanbase +{ +using namespace lib; + +namespace common +{ + +ObTenantErrsimEvent::ObTenantErrsimEvent() + : timestamp_(0), + type_(), + errsim_error_(OB_SUCCESS), + backtrace_() +{ +} + +void ObTenantErrsimEvent::reset() +{ + timestamp_ = 0; + type_.reset(); + errsim_error_ = OB_SUCCESS; + backtrace_.reset(); +} + +bool ObTenantErrsimEvent::is_valid() const +{ + return timestamp_ > 0 && type_.is_valid() && !backtrace_.is_empty(); +} + +void ObTenantErrsimEvent::build_event(const int32_t result) +{ + timestamp_ = ObTimeUtil::current_time(); + errsim_error_ = result; + lbt(backtrace_.ptr(), backtrace_.capacity()); +#ifdef ERRSIM + type_ = THIS_WORKER.get_module_type(); +#else + ObErrsimModuleType tmp_type(ObErrsimModuleType::ERRSIM_MODULE_NONE); + type_ = tmp_type; +#endif +} + +} //common +} //oceanbase diff --git a/deps/oblib/src/common/errsim_module/ob_tenant_errsim_event.h b/deps/oblib/src/common/errsim_module/ob_tenant_errsim_event.h new file mode 100644 index 0000000000..e6c49e289f --- /dev/null +++ b/deps/oblib/src/common/errsim_module/ob_tenant_errsim_event.h @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_COMMON_ERRSIM_OB_TENANT_ERRSIM_MODULE_MGR_H_ +#define SRC_COMMON_ERRSIM_OB_TENANT_ERRSIM_MODULE_MGR_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" +#include "ob_errsim_module_type.h" +#include "lib/hash/ob_hashset.h" +#include "lib/hash/ob_hashmap.h" +#include "lib/lock/ob_bucket_lock.h" +#include "lib/utility/ob_backtrace.h" + +namespace oceanbase +{ +namespace common +{ + +struct ObTenantErrsimEvent final +{ + ObTenantErrsimEvent(); + ~ObTenantErrsimEvent() = default; + void reset(); + bool is_valid() const; + void build_event(const int32_t result); + + typedef ObFixedLengthString Lbt; + + TO_STRING_KV(K_(timestamp), K_(type), K_(errsim_error), K_(backtrace)); + int64_t timestamp_; + ObErrsimModuleType type_; + int32_t errsim_error_; + Lbt backtrace_; +}; + +} // namespace common +} // namespace oceanbase +#endif diff --git a/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp b/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp index 093a815eda..c4d713054d 100644 --- a/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp +++ b/deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp @@ -23,6 +23,7 @@ #include "lib/oblog/ob_log.h" #include "common/ob_smart_var.h" #include "rpc/obrpc/ob_rpc_packet.h" +#include "common/errsim_module/ob_errsim_module_interface.h" using namespace oceanbase::lib; using namespace oceanbase::common; @@ -401,16 +402,33 @@ void* ObTenantCtxAllocator::common_alloc(const int64_t size, const ObMemAttr &at { SANITY_DISABLE_CHECK_RANGE(); // prevent sanity_check_range void *ret = nullptr; + AObject *obj = nullptr; + bool sample_allowed = false; + bool is_errsim = false; if (!attr.label_.is_valid()) { LIB_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "OB_MOD_DO_NOT_USE_ME ALLOC", K(size)); } - bool sample_allowed = ObMallocSampleLimiter::malloc_sample_allowed(size, attr); - const int64_t alloc_size = sample_allowed ? (size + AOBJECT_BACKTRACE_SIZE) : size; - AObject *obj = allocator.alloc_object(alloc_size, attr); - if (OB_ISNULL(obj) && g_alloc_failed_ctx().need_wash()) { - int64_t total_size = ta.sync_wash(); - obj = allocator.alloc_object(alloc_size, attr); + +#ifdef ERRSIM + const ObErrsimModuleType type = THIS_WORKER.get_module_type(); + if (is_errsim_module(ta.get_tenant_id(), type.type_)) { + //errsim alloc memory failed. + obj = nullptr; + is_errsim = true; } +#endif + + if (OB_UNLIKELY(is_errsim)) { + } else { + sample_allowed = ObMallocSampleLimiter::malloc_sample_allowed(size, attr); + const int64_t alloc_size = sample_allowed ? (size + AOBJECT_BACKTRACE_SIZE) : size; + obj = allocator.alloc_object(alloc_size, attr); + if (OB_ISNULL(obj) && g_alloc_failed_ctx().need_wash()) { + int64_t total_size = ta.sync_wash(); + obj = allocator.alloc_object(alloc_size, attr); + } + } + if (NULL != obj) { obj->on_malloc_sample_ = sample_allowed; ob_malloc_sample_backtrace(obj, size); @@ -425,7 +443,7 @@ void* ObTenantCtxAllocator::common_alloc(const int64_t size, const ObMemAttr &at int level = ObFreeLogPrinter::get_level(); ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_, attr.ctx_id_, level); - const char *msg = alloc_failed_msg(); + const char *msg = is_errsim ? "[ERRSIM] errsim inject memory error" : alloc_failed_msg(); LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg)); _OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, " "ctx_limit=%ld, tenant_hold=%ld, tenant_limit=%ld", @@ -448,7 +466,10 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size, if (!attr.label_.is_valid()) { LIB_LOG_RET(ERROR, OB_INVALID_ARGUMENT, "OB_MOD_DO_NOT_USE_ME REALLOC", K(size)); } + AObject *obj = NULL; + bool sample_allowed = false; + bool is_errsim = false; if (NULL != ptr) { obj = reinterpret_cast((char*)ptr - AOBJECT_HEADER_SIZE); abort_unless(obj->is_valid()); @@ -458,13 +479,27 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size, SANITY_POISON(obj->data_, obj->alloc_bytes_); get_mem_leak_checker().on_free(*obj); } - bool sample_allowed = ObMallocSampleLimiter::malloc_sample_allowed(size, attr); - const int64_t alloc_size = sample_allowed ? (size + AOBJECT_BACKTRACE_SIZE) : size; - obj = allocator.realloc_object(obj, alloc_size, attr); - if (OB_ISNULL(obj) && g_alloc_failed_ctx().need_wash()) { - int64_t total_size = ta.sync_wash(); - obj = allocator.realloc_object(obj, alloc_size, attr); + +#ifdef ERRSIM + const ObErrsimModuleType type = THIS_WORKER.get_module_type(); + if (is_errsim_module(ta.get_tenant_id(), type.type_)) { + //errsim alloc memory failed. + obj = nullptr; + is_errsim = true; } +#endif + + if (OB_UNLIKELY(is_errsim)) { + } else { + sample_allowed = ObMallocSampleLimiter::malloc_sample_allowed(size, attr); + const int64_t alloc_size = sample_allowed ? (size + AOBJECT_BACKTRACE_SIZE) : size; + obj = allocator.realloc_object(obj, alloc_size, attr); + if(OB_ISNULL(obj) && g_alloc_failed_ctx().need_wash()) { + int64_t total_size = ta.sync_wash(); + obj = allocator.realloc_object(obj, alloc_size, attr); + } + } + if (obj != NULL) { obj->on_malloc_sample_ = sample_allowed; ob_malloc_sample_backtrace(obj, size); @@ -478,7 +513,7 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size, int level = ObFreeLogPrinter::get_level(); ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_, attr.ctx_id_, level); - const char *msg = alloc_failed_msg(); + const char *msg = is_errsim ? "[ERRSIM] errsim inject memory error" : alloc_failed_msg(); LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg)); _OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, " "ctx_limit=%ld, tenant_hold=%ld, tenant_limit=%ld", diff --git a/deps/oblib/src/lib/worker.cpp b/deps/oblib/src/lib/worker.cpp index 8e7ade4d42..6501f0c18a 100644 --- a/deps/oblib/src/lib/worker.cpp +++ b/deps/oblib/src/lib/worker.cpp @@ -22,7 +22,12 @@ using namespace oceanbase::common; using namespace oceanbase::lib; -OB_SERIALIZE_MEMBER(ObRuntimeContext, compat_mode_); +#ifdef ERRSIM + OB_SERIALIZE_MEMBER(ObRuntimeContext, compat_mode_, module_type_); +#else + OB_SERIALIZE_MEMBER(ObRuntimeContext, compat_mode_); +#endif + namespace oceanbase { namespace lib { diff --git a/deps/oblib/src/lib/worker.h b/deps/oblib/src/lib/worker.h index 65de9e35d3..ef692e0baf 100644 --- a/deps/oblib/src/lib/worker.h +++ b/deps/oblib/src/lib/worker.h @@ -17,6 +17,7 @@ #include "lib/ob_define.h" #include "lib/rc/context.h" #include "lib/runtime.h" +#include "common/errsim_module/ob_errsim_module_type.h" namespace oceanbase { @@ -115,6 +116,11 @@ public: static Worker& self(); static void set_worker_to_thread_local(Worker *worker); +#ifdef ERRSIM + static void set_module_type(const ObErrsimModuleType &module_type); + static ObErrsimModuleType get_module_type(); +#endif + public: static __thread Worker *self_; @@ -213,6 +219,29 @@ private: Worker::CompatMode last_compat_mode_; }; +#ifdef ERRSIM +//set current errsim module in code snippet and set last errsim module when guard destructor +class ErrsimModuleGuard final +{ +public: + ErrsimModuleGuard(ObErrsimModuleType::TYPE type) + { + last_type_ = THIS_WORKER.get_module_type().type_; + ObErrsimModuleType curr_type(type); + THIS_WORKER.set_module_type(curr_type); + } + + ~ErrsimModuleGuard() + { + ObErrsimModuleType curr_type(last_type_); + THIS_WORKER.set_module_type(curr_type); + } + +private: + ObErrsimModuleType::TYPE last_type_; +}; +#endif + // used to check compatibility mode. class ObRuntimeContext { @@ -222,6 +251,9 @@ public: : compat_mode_(Worker::CompatMode::MYSQL) {} Worker::CompatMode compat_mode_; +#ifdef ERRSIM + ObErrsimModuleType module_type_; +#endif }; inline ObRuntimeContext &get_ob_runtime_context() @@ -259,6 +291,19 @@ OB_INLINE Worker::CompatMode Worker::get_compatibility_mode() return get_compat_mode(); } +#ifdef ERRSIM +OB_INLINE void Worker::set_module_type(const ObErrsimModuleType &module_type) +{ + get_ob_runtime_context().module_type_ = module_type; +} + +OB_INLINE ObErrsimModuleType Worker::get_module_type() +{ + return get_ob_runtime_context().module_type_; +} +#endif + + } // end of namespace lib } // end of namespace oceanbase diff --git a/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp b/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp index 823616ac90..aeb65b78c9 100644 --- a/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp +++ b/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp @@ -120,6 +120,10 @@ int ObReqQueue::process_task(void *task) } else { ObCurTraceId::set(trace_id); } + +#ifdef ERRSIM + THIS_WORKER.set_module_type(packet.get_module_type()); +#endif // Do not set thread local log level while log level upgrading (OB_LOGGER.is_info_as_wdiag) if (OB_LOGGER.is_info_as_wdiag()) { ObThreadLogLevelUtils::clear(); diff --git a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp index ec8ea4b311..e033771caa 100644 --- a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp @@ -129,7 +129,12 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz) ucb_->on_invalid(); RPC_LOG(WARN, "rpc_decode_ob_packet fail", K(ret)); } else if (OB_FALSE_IT(ObCurTraceId::set(ret_pkt->get_trace_id()))) { - } else if (OB_FAIL(ucb_->decode(ret_pkt))) { + } +#ifdef ERRSIM + else if (OB_FALSE_IT(THIS_WORKER.set_module_type(ret_pkt->get_module_type()))) { + } +#endif + else if (OB_FAIL(ucb_->decode(ret_pkt))) { ucb_->on_invalid(); RPC_LOG(WARN, "ucb.decode fail", K(ret)); } else { diff --git a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_server.cpp b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_server.cpp index b3e7d9b57d..73acb0c996 100644 --- a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_server.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_server.cpp @@ -66,6 +66,11 @@ int ObPocServerHandleContext::create(int64_t resp_id, const char* buf, int64_t s timeguard.click(); ObRpcMemPool* pool = ObRpcMemPool::create(tenant_id, pcode_label, pool_size); void *temp = NULL; + +#ifdef ERRSIM + THIS_WORKER.set_module_type(tmp_pkt.get_module_type()); +#endif + if (OB_ISNULL(pool)) { ret = common::OB_ALLOCATE_MEMORY_FAILED; RPC_LOG(WARN, "create memory pool failed", K(tenant_id), K(pcode_label)); @@ -154,6 +159,9 @@ int ObPocServerHandleContext::resp_error(uint64_t resp_id, int err_code, const c res_pkt.set_dst_cluster_id(recv_pkt.get_src_cluster_id()); int64_t receive_ts = ObTimeUtility::current_time(); res_pkt.set_request_arrival_time(receive_ts); +#ifdef ERRSIM + res_pkt.set_module_type(recv_pkt.get_module_type()); +#endif } } res_pkt.set_resp(); diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_packet.cpp index 09b8ec522c..014e19b45b 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet.cpp @@ -85,6 +85,11 @@ int ObRpcPacketHeader::serialize(char* buf, const int64_t buf_len, int64_t& pos) LOG_WARN("Encode error", K(ret), KP(buf), K(buf_len), K(pos)); } else { //do nothing +#ifdef ERRSIM + if (OB_FAIL(encode_i64(buf, buf_len, pos, static_cast(module_type_.type_)))) { + LOG_WARN("Encode error", K(ret), KP(buf), K(buf_len), K(pos)); + } +#endif } } else { ret = OB_BUF_NOT_ENOUGH; @@ -165,6 +170,15 @@ int ObRpcPacketHeader::deserialize(const char* buf, const int64_t data_len, int6 } else if (hlen_ > pos && OB_FAIL(decode_i64(buf, hlen_, pos, reinterpret_cast(&cluster_name_hash_)))) { LOG_WARN("Decode error", K(ret), KP(buf), K(hlen_), K(pos)); } else { +#ifdef ERRSIM + int64_t type = 0; + if (hlen_ > pos && OB_FAIL(decode_i64(buf, hlen_, pos, &type))) { + LOG_WARN("Decode error", K(ret), KP(buf), K(hlen_), K(pos)); + } else { + module_type_.type_ = static_cast(type); + } +#endif + } ObSequence::update_max_seq_no(seq_no_); LOG_DEBUG("rpc receive seq_no ", K_(seq_no), K(ObSequence::get_max_seq_no())); diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h index ea16351414..a469410726 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet.h @@ -20,6 +20,7 @@ #include "lib/compress/ob_compressor_pool.h" #include "rpc/obrpc/ob_rpc_time.h" #include "rpc/ob_packet.h" +#include "common/errsim_module/ob_errsim_module_type.h" namespace oceanbase { @@ -138,7 +139,11 @@ private: class ObRpcPacketHeader { public: +#ifdef ERRSIM + static const uint8_t HEADER_SIZE = 144; // add 8 bit for errsim module +#else static const uint8_t HEADER_SIZE = 136; // 112 -> 128: add 16 bytes for trace_id ipv6 extension. (Note yanyuan.cxf: but you should never change it) +#endif static const uint16_t RESP_FLAG = 1 << 15; static const uint16_t STREAM_FLAG = 1 << 14; static const uint16_t STREAM_LAST_FLAG = 1 << 13; @@ -173,6 +178,10 @@ public: int32_t group_id_; uint64_t cluster_name_hash_; +#ifdef ERRSIM + ObErrsimModuleType module_type_; +#endif + int serialize(char* buf, const int64_t buf_len, int64_t& pos); int deserialize(const char* buf, const int64_t data_len, int64_t& pos); @@ -335,6 +344,11 @@ public: } static uint64_t get_self_cluster_name_hash(); +#ifdef ERRSIM + inline void set_module_type(const ObErrsimModuleType &module_type); + inline const ObErrsimModuleType get_module_type() const; +#endif + private: ObRpcPacketHeader hdr_; const char *cdata_; @@ -855,6 +869,18 @@ inline ObRpcCheckSumCheckLevel get_rpc_checksum_check_level_from_string( return ret_type; } +#ifdef ERRSIM +void ObRpcPacket::set_module_type(const ObErrsimModuleType &module_type) +{ + hdr_.module_type_ = module_type; +} + +const ObErrsimModuleType ObRpcPacket::get_module_type() const +{ + return hdr_.module_type_; +} +#endif + } // end of namespace rpc } // end of namespace oceanbase diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp index 7b5e697f4e..e2691b76f7 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_processor_base.cpp @@ -294,6 +294,10 @@ int ObRpcProcessorBase::do_response(const Response &rsp) // The cluster_id of the response must be the src_cluster_id of the request packet->set_dst_cluster_id(rpc_pkt_->get_src_cluster_id()); +#ifdef ERRSIM + packet->set_module_type(THIS_WORKER.get_module_type()); +#endif + packet->set_request_arrival_time(req_->get_request_arrival_time()); packet->set_arrival_push_diff(req_->get_arrival_push_diff()); packet->set_push_pop_diff(req_->get_push_pop_diff()); diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp index de00964705..8096c3e3e9 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp @@ -153,6 +153,10 @@ int ObRpcProxy::init_pkt( pkt->set_trace_id(common::ObCurTraceId::get()); } +#ifdef ERRSIM + pkt->set_module_type(THIS_WORKER.get_module_type()); +#endif + if (OB_SUCC(ret)) { pkt->set_pcode(pcode); //Assign a channel id to this new packet diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_request.cpp b/deps/oblib/src/rpc/obrpc/ob_rpc_request.cpp index a386fa85d9..146c9bff4f 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_request.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_request.cpp @@ -89,6 +89,9 @@ int ObRpcRequest::do_flush_buffer(common::ObDataBuffer *buffer, const ObRpcPacke res_packet->set_session_id(sess_id); res_packet->set_trace_id(common::ObCurTraceId::get()); res_packet->set_resp(); +#ifdef ERRSIM + res_packet->set_module_type(THIS_WORKER.get_module_type()); +#endif res_packet->calc_checksum(); } RPC_REQ_OP.response_result(this, res_packet); diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 9b6ca5d9f8..5ce9a37cf9 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -130,6 +130,10 @@ #include "rootserver/ob_rs_event_history_table_operator.h" #include "rootserver/ob_heartbeat_service.h" #include "share/detect/ob_detect_manager.h" +#ifdef ERRSIM +#include "share/errsim_module/ob_tenant_errsim_module_mgr.h" +#include "share/errsim_module/ob_tenant_errsim_event_mgr.h" +#endif using namespace oceanbase; using namespace oceanbase::lib; @@ -513,6 +517,12 @@ int ObMultiTenant::init(ObAddr myaddr, // mtl_wait_default, mtl_destroy_default); } MTL_BIND2(mtl_new_default, rootserver::ObHeartbeatService::mtl_init, nullptr, rootserver::ObHeartbeatService::mtl_stop, rootserver::ObHeartbeatService::mtl_wait, mtl_destroy_default); + +#ifdef ERRSIM + MTL_BIND2(mtl_new_default, ObTenantErrsimModuleMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); + MTL_BIND2(mtl_new_default, ObTenantErrsimEventMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); +#endif + } if (OB_SUCC(ret)) { @@ -921,7 +931,6 @@ int ObMultiTenant::create_tenant(const ObTenantMeta &meta, bool write_slog, cons LOG_WARN("fail to set_tenant_mem_limit", K(ret), K(tenant_id)); } } - if (OB_SUCC(ret)) { if (write_slog && OB_FAIL(write_create_tenant_commit_slog(tenant_id))) { LOG_ERROR("fail to write create tenant commit slog", K(ret), K(tenant_id)); diff --git a/src/observer/omt/ob_tenant_config.cpp b/src/observer/omt/ob_tenant_config.cpp index 82ae3857e2..2afc12ef39 100644 --- a/src/observer/omt/ob_tenant_config.cpp +++ b/src/observer/omt/ob_tenant_config.cpp @@ -23,6 +23,7 @@ #include "observer/omt/ob_tenant_config.h" #include "observer/omt/ob_tenant_config_mgr.h" #include "sql/monitor/flt/ob_flt_control_info_mgr.h" +#include "share/errsim_module/ob_errsim_module_interface_imp.h" using namespace oceanbase::common; @@ -300,6 +301,11 @@ int ObTenantConfig::update_local(int64_t expected_version, ObMySQLProxy::MySQLRe } else if (OB_FAIL(publish_special_config_after_dump())) { LOG_WARN("publish special config after dump failed", K(tenant_id_), K(ret)); } +#ifdef ERRSIM + else if (OB_FAIL(build_errsim_module_())) { + LOG_WARN("failed to build errsim module", K(ret), K(tenant_id_)); + } +#endif print(); } else { LOG_WARN("Read tenant config from inner table error", K_(tenant_id), K(ret)); @@ -508,5 +514,36 @@ OB_DEF_SERIALIZE_SIZE(ObTenantConfig) return len; } +#ifdef ERRSIM +int ObTenantConfig::build_errsim_module_() +{ + int ret = OB_SUCCESS; + char buf[ObErrsimModuleTypeHelper::MAX_TYPE_NAME_LENGTH] = ""; + ObTenantErrsimModuleMgr::ModuleArray module_array; + ObTenantErrsimModuleMgr::ErrsimModuleString string; + + for (int64_t i = 0; OB_SUCC(ret) && i < this->errsim_module_types.size(); ++i) { + if (OB_FAIL(this->errsim_module_types.get( + static_cast(i), buf, sizeof(buf)))) { + LOG_WARN("get rs failed", K(ret), K(i)); + } else if (OB_FAIL(string.assign(buf))) { + LOG_WARN("failed to assign buffer", K(ret)); + } else if (OB_FAIL(module_array.push_back(string))) { + LOG_WARN("failed to push string into array", K(ret), K(string)); + } + } + + if (OB_SUCC(ret)) { + const int64_t percentage = this->errsim_module_error_percentage; + + if (build_tenant_errsim_moulde(tenant_id_, current_version_, module_array, percentage)) { + LOG_WARN("failed to build tenant module", K(ret), K(tenant_id_)); + } + } + return ret; +} +#endif + + } // omt } // oceanbase diff --git a/src/observer/omt/ob_tenant_config.h b/src/observer/omt/ob_tenant_config.h index c6db65f146..9f5eba138f 100644 --- a/src/observer/omt/ob_tenant_config.h +++ b/src/observer/omt/ob_tenant_config.h @@ -96,6 +96,10 @@ public: bool check_unit = true); OB_UNIS_VERSION(1); +private: +#ifdef ERRSIM + int build_errsim_module_(); +#endif private: uint64_t tenant_id_; int64_t current_version_; // 当前 tenant config 正在被 task 更新中的版本 diff --git a/src/observer/omt/ob_worker_processor.cpp b/src/observer/omt/ob_worker_processor.cpp index 1ee4b4992a..5feab7bc5e 100644 --- a/src/observer/omt/ob_worker_processor.cpp +++ b/src/observer/omt/ob_worker_processor.cpp @@ -111,6 +111,11 @@ int ObWorkerProcessor::process(rpc::ObRequest &req) = static_cast(req.get_packet()); NG_TRACE_EXT(start_rpc, OB_ID(addr), RPC_REQ_OP.get_peer(&req), OB_ID(pcode), packet.get_pcode()); ObCurTraceId::set(req.generate_trace_id(myaddr_)); + +#ifdef ERRSIM + THIS_WORKER.set_module_type(packet.get_module_type()); +#endif + // Do not set thread local log level while log level upgrading (OB_LOGGER.is_info_as_wdiag) if (OB_LOGGER.is_info_as_wdiag()) { ObThreadLogLevelUtils::clear(); diff --git a/src/observer/table/ob_rpc_async_response.h b/src/observer/table/ob_rpc_async_response.h index 5cf6e4205c..b5b1aee043 100644 --- a/src/observer/table/ob_rpc_async_response.h +++ b/src/observer/table/ob_rpc_async_response.h @@ -105,6 +105,10 @@ int ObRpcAsyncResponse::do_response(ObRpcPacket *response_pkt, bool bad_routi packet->set_trace_id(rpc_pkt->get_trace_id()); packet->set_resp(); +#ifdef ERRSIM + packet->set_module_type(rpc_pkt->get_module_type()); +#endif + packet->set_request_arrival_time(req_->get_request_arrival_time()); packet->set_arrival_push_diff(req_->get_arrival_push_diff()); packet->set_push_pop_diff(req_->get_push_pop_diff()); diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index 1ded37bb0d..16ca3f90d2 100755 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -484,6 +484,13 @@ ob_set_subtarget(ob_share balance balance/ob_balance_task_helper_operator.cpp ) +if (OB_ERRSIM) + ob_set_subtarget(ob_share errsim_module + errsim_module/ob_tenant_errsim_module_mgr.cpp + errsim_module/ob_tenant_errsim_event_mgr.cpp + errsim_module/ob_errsim_module_interface_imp.cpp + ) +endif() ob_add_new_object_target(ob_share ob_share) diff --git a/src/share/errsim_module/ob_errsim_module_interface_imp.cpp b/src/share/errsim_module/ob_errsim_module_interface_imp.cpp new file mode 100644 index 0000000000..eb281ebd28 --- /dev/null +++ b/src/share/errsim_module/ob_errsim_module_interface_imp.cpp @@ -0,0 +1,110 @@ + +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#define USING_LOG_PREFIX COMMON +#include "ob_errsim_module_interface_imp.h" +#include +#include +#include +#include "lib/ob_define.h" +#include "share/rc/ob_tenant_base.h" +#include "ob_tenant_errsim_event_mgr.h" + +using namespace oceanbase::share; +namespace oceanbase { +namespace common { + +int build_tenant_errsim_moulde( + const uint64_t tenant_id, + const int64_t config_version, + const common::ObArray> &module_array, + const int64_t percentage) +{ + int ret = OB_SUCCESS; + const uint64_t tmp_tenant_id = is_virtual_tenant_id(tenant_id) ? MTL_ID() : tenant_id; + + if (OB_INVALID_ID == tmp_tenant_id || config_version < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("build tenant module get invalid argument", K(ret), K(tmp_tenant_id), K(config_version)); + } else if (is_virtual_tenant_id(tmp_tenant_id)) { + //do nothing + } else { + MTL_SWITCH(tmp_tenant_id) { + ObTenantErrsimModuleMgr *errsim_module_mgr = nullptr; + if (OB_ISNULL(errsim_module_mgr = MTL(ObTenantErrsimModuleMgr *))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "errsim module mgr should not be NULL", K(ret), KP(errsim_module_mgr)); + } else if (OB_FAIL(errsim_module_mgr->build_tenant_moulde(tmp_tenant_id, config_version, module_array, percentage))) { + LOG_WARN("failed to build tenant module", K(ret), K(tmp_tenant_id), K(config_version)); + } + } + } + return ret; +} + +bool is_errsim_module( + const uint64_t tenant_id, + const ObErrsimModuleType::TYPE &type) +{ + bool b_ret = false; + int ret = OB_SUCCESS; + const uint64_t tmp_tenant_id = is_virtual_tenant_id(tenant_id) ? MTL_ID() : tenant_id; + if (OB_INVALID_ID == tmp_tenant_id || !ObErrsimModuleTypeHelper::is_valid(type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("is errsim module get invalid argument", K(ret), K(tenant_id), K(tmp_tenant_id), K(type)); + } else if (is_virtual_tenant_id(tmp_tenant_id)) { + b_ret = false; + } else if (ObErrsimModuleType::ERRSIM_MODULE_NONE == type) { + b_ret = false; + } else { + MTL_SWITCH(tmp_tenant_id) { + ObTenantErrsimModuleMgr *errsim_module_mgr = nullptr; + if (OB_ISNULL(errsim_module_mgr = MTL(ObTenantErrsimModuleMgr *))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "errsim module mgr should not be NULL", K(ret), KP(errsim_module_mgr)); + } else { + b_ret = errsim_module_mgr->is_errsim_module(type); + } + } + } + return b_ret; +} + +int add_tenant_errsim_event( + const uint64_t tenant_id, + const ObTenantErrsimEvent &event) +{ + bool b_ret = false; + int ret = OB_SUCCESS; + const uint64_t tmp_tenant_id = is_virtual_tenant_id(tenant_id) ? MTL_ID() : tenant_id; + if (OB_INVALID_ID == tmp_tenant_id || !event.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("is errsim module get invalid argument", K(ret), K(tmp_tenant_id), K(event)); + } else if (is_virtual_tenant_id(tmp_tenant_id)) { + //do nothing + } else { + MTL_SWITCH(tmp_tenant_id) { + ObTenantErrsimEventMgr *errsim_event_mgr = nullptr; + if (OB_ISNULL(errsim_event_mgr = MTL(ObTenantErrsimEventMgr *))) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "errsim event mgr should not be NULL", K(ret), KP(errsim_event_mgr)); + } else if (OB_FAIL(errsim_event_mgr->add_tenant_event(event))) { + LOG_WARN("failed to add tenant event", K(ret), K(event)); + } + } + } + return b_ret; +} + + +} // common +} // oceanbase diff --git a/src/share/errsim_module/ob_errsim_module_interface_imp.h b/src/share/errsim_module/ob_errsim_module_interface_imp.h new file mode 100644 index 0000000000..6bc3684647 --- /dev/null +++ b/src/share/errsim_module/ob_errsim_module_interface_imp.h @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_SHARE_ERRSIM_MODULE_OB_TENANT_ERRSIM_MODULE_INTERFACE_IMP_H_ +#define SRC_SHARE_ERRSIM_MODULE_OB_TENANT_ERRSIM_MODULE_INTERFACE_IMP_H_ + +#include "ob_tenant_errsim_module_mgr.h" +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" +#include "common/errsim_module/ob_tenant_errsim_event.h" + + +namespace oceanbase +{ +namespace common +{ + +int build_tenant_errsim_moulde( + const uint64_t tenant_id, + const int64_t config_version, + const common::ObArray> &module_array, + const int64_t percentage); +bool is_errsim_module( + const uint64_t tenant_id, + const ObErrsimModuleType::TYPE &type); +int add_tenant_errsim_event( + const uint64_t tenant_id, + const ObTenantErrsimEvent &event); + + +} // namespace common +} // namespace oceanbase +#endif diff --git a/src/share/errsim_module/ob_tenant_errsim_event_mgr.cpp b/src/share/errsim_module/ob_tenant_errsim_event_mgr.cpp new file mode 100644 index 0000000000..f2da19e59f --- /dev/null +++ b/src/share/errsim_module/ob_tenant_errsim_event_mgr.cpp @@ -0,0 +1,81 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_tenant_errsim_event_mgr.h" +#include "share/rc/ob_tenant_base.h" + +namespace oceanbase +{ +using namespace lib; + +namespace share +{ + +ObTenantErrsimEventMgr::ObTenantErrsimEventMgr() + : is_inited_(false), + lock_(), + event_array_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator("TErrsimEvent", MTL_ID())) +{ +} + +ObTenantErrsimEventMgr::~ObTenantErrsimEventMgr() +{ +} + +int ObTenantErrsimEventMgr::mtl_init(ObTenantErrsimEventMgr *&errsim_event_mgr) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(errsim_event_mgr->init())) { + LOG_WARN("failed to init errsim event mgr", K(ret), KP(errsim_event_mgr)); + } + return ret; +} + +void ObTenantErrsimEventMgr::destroy() +{ + event_array_.destroy(); +} + +int ObTenantErrsimEventMgr::init() +{ + int ret = OB_SUCCESS; + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("tenant errsim event mgr init twice", K(ret)); + } else { + is_inited_ = true; + } + return ret; +} + +int ObTenantErrsimEventMgr::add_tenant_event( + const ObTenantErrsimEvent &event) +{ + int ret = OB_SUCCESS; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("tenant errsim event mgr do not init", K(ret)); + } else if (!event.is_valid()) { + LOG_WARN("add tenant event get invalid argument", K(ret), K(event)); + } else { + common::SpinWLockGuard guard(lock_); + if (OB_FAIL(event_array_.push_back(event))) { + LOG_WARN("failed to add tenant event", K(ret), K(event)); + } + } + return ret; +} + + +} //share +} //oceanbase diff --git a/src/share/errsim_module/ob_tenant_errsim_event_mgr.h b/src/share/errsim_module/ob_tenant_errsim_event_mgr.h new file mode 100644 index 0000000000..e07b508bd8 --- /dev/null +++ b/src/share/errsim_module/ob_tenant_errsim_event_mgr.h @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_SHARE_ERRSIM_OB_TENANT_ERRSIM_MODULE_MGR_H_ +#define SRC_SHARE_ERRSIM_OB_TENANT_ERRSIM_MODULE_MGR_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" +#include "common/errsim_module/ob_errsim_module_type.h" +#include "lib/hash/ob_hashset.h" +#include "lib/hash/ob_hashmap.h" +#include "lib/lock/ob_bucket_lock.h" +#include "lib/utility/ob_backtrace.h" +#include "common/errsim_module/ob_tenant_errsim_event.h" + +namespace oceanbase +{ +namespace share +{ + +class ObTenantErrsimEventMgr +{ +public: + ObTenantErrsimEventMgr(); + virtual ~ObTenantErrsimEventMgr(); + static int mtl_init(ObTenantErrsimEventMgr *&errsim_module_mgr); + int init(); + int add_tenant_event( + const ObTenantErrsimEvent &event); + void destroy(); +private: + bool is_inited_; + common::SpinRWLock lock_; + ObArray event_array_; + DISALLOW_COPY_AND_ASSIGN(ObTenantErrsimEventMgr); +}; + +} // namespace share +} // namespace oceanbase +#endif diff --git a/src/share/errsim_module/ob_tenant_errsim_module_mgr.cpp b/src/share/errsim_module/ob_tenant_errsim_module_mgr.cpp new file mode 100644 index 0000000000..15274a2d79 --- /dev/null +++ b/src/share/errsim_module/ob_tenant_errsim_module_mgr.cpp @@ -0,0 +1,160 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX COMMON +#include "ob_tenant_errsim_module_mgr.h" +#include "share/rc/ob_tenant_base.h" + +namespace oceanbase +{ +using namespace lib; + +namespace share +{ + +ObTenantErrsimModuleMgr::ObTenantErrsimModuleMgr() + : is_inited_(false), + tenant_id_(OB_INVALID_ID), + lock_(), + config_version_(0), + is_whole_module_(false), + module_set_(), + percentage_(0) +{ +} + +ObTenantErrsimModuleMgr::~ObTenantErrsimModuleMgr() +{ +} + +int ObTenantErrsimModuleMgr::mtl_init(ObTenantErrsimModuleMgr *&errsim_module_mgr) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(errsim_module_mgr->init(tenant_id))) { + LOG_WARN("failed to init errsim module mgr", K(ret), KP(errsim_module_mgr)); + } + return ret; +} + +void ObTenantErrsimModuleMgr::destroy() +{ + module_set_.destroy(); +} + +int ObTenantErrsimModuleMgr::init(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + const ObMemAttr bucket_attr(tenant_id, "ErrsimModuleSet"); + + if (is_inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("tenant errsim module mgr init twice", K(ret)); + } else if (OB_INVALID_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("init tenant errsim module mgr get invalid argument", K(ret), K(tenant_id)); + } else if (OB_FAIL(module_set_.create(MAX_BUCKET_NUM, bucket_attr))) { + LOG_WARN("failed to create module set", K(ret), K(tenant_id)); + } else { + tenant_id_ = tenant_id; + is_whole_module_ = false; + is_inited_ = true; + } + return ret; +} + +bool ObTenantErrsimModuleMgr::is_errsim_module( + const ObErrsimModuleType::TYPE &type) +{ + bool b_ret = false; + int ret = OB_SUCCESS; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("tenant errsim module mgr do not init", K(ret)); + } else if (!ObErrsimModuleTypeHelper::is_valid(type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("is errsim module get invalid argument", K(ret), K(type)); + } else { + common::SpinRLockGuard guard(lock_); + const int64_t percentage = ObRandom::rand(0, 100); + if (percentage > percentage_) { + b_ret = false; + } else if (is_whole_module_) { + b_ret = true; + } else { + ObErrsimModuleType module_type(type); + const int32_t hash_ret = module_set_.exist_refactored(module_type); + if (OB_HASH_NOT_EXIST == hash_ret) { + b_ret = false; + } else if (OB_HASH_EXIST == hash_ret) { + b_ret = true; + } else { + b_ret = false; + LOG_ERROR("failed to check module type exist", K(hash_ret)); + } + } + } + return b_ret; +} + +int ObTenantErrsimModuleMgr::build_tenant_moulde( + const uint64_t tenant_id, + const int64_t config_version, + const ModuleArray &module_array, + const int64_t percentage) +{ + int ret = OB_SUCCESS; + char type_buf[ObErrsimModuleTypeHelper::MAX_TYPE_NAME_LENGTH] = ""; + ObErrsimModuleType::TYPE type = ObErrsimModuleType::ERRSIM_MODULE_MAX; + const int32_t flag = 1; + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("tenant errsim module mgr do not init", K(ret)); + } else if (OB_INVALID_ID == tenant_id || config_version < 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("build tenant module get invalid argument", K(ret), K(tenant_id), K(config_version)); + } else { + common::SpinWLockGuard guard(lock_); + if (config_version <= config_version_) { + //do nothing + } else { + is_whole_module_ = false; + module_set_.reuse(); + percentage_ = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < module_array.size(); ++i) { + const ErrsimModuleString &string = module_array.at(i); + type = ObErrsimModuleTypeHelper::get_type(string.ptr()); + ObErrsimModuleType module_type(type); + if (!module_type.is_valid()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("errsim module type is unexpected", K(ret), K(module_type), K(type_buf)); + } else if (ObErrsimModuleType::ERRSIM_MODULE_ALL == module_type.type_) { + is_whole_module_ = true; + } else if (OB_FAIL(module_set_.set_refactored(module_type, flag))) { + LOG_WARN("failed to set module set", K(ret), K(module_type)); + } else { + LOG_INFO("succeed set module", K(module_type), K(tenant_id)); + } + } + + if (OB_SUCC(ret)) { + percentage_ = percentage; + } + } + } + return ret; +} + + +} //share +} //oceanbase diff --git a/src/share/errsim_module/ob_tenant_errsim_module_mgr.h b/src/share/errsim_module/ob_tenant_errsim_module_mgr.h new file mode 100644 index 0000000000..86bc3db6f0 --- /dev/null +++ b/src/share/errsim_module/ob_tenant_errsim_module_mgr.h @@ -0,0 +1,62 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef SRC_SHARE_ERRSIM_MODULE_OB_TENANT_ERRSIM_MODULE_MGR_H_ +#define SRC_SHARE_ERRSIM_MODULE_OB_TENANT_ERRSIM_MODULE_MGR_H_ + +#include "lib/ob_define.h" +#include "lib/utility/ob_print_utils.h" +#include "common/errsim_module/ob_errsim_module_type.h" +#include "lib/hash/ob_hashset.h" +#include "lib/hash/ob_hashmap.h" +#include "lib/lock/ob_bucket_lock.h" + +namespace oceanbase +{ +namespace share +{ + +class ObTenantErrsimModuleMgr +{ +public: + typedef ObFixedLengthString ErrsimModuleString; + typedef ObArray> ModuleArray; +public: + ObTenantErrsimModuleMgr(); + virtual ~ObTenantErrsimModuleMgr(); + static int mtl_init(ObTenantErrsimModuleMgr *&errsim_module_mgr); + int init(const uint64_t tenant_id); + int build_tenant_moulde( + const uint64_t tenant_id, + const int64_t config_version, + const ModuleArray &module_array, + const int64_t percentage); + bool is_errsim_module( + const ObErrsimModuleType::TYPE &type); + void destroy(); + +private: + typedef hash::ObHashSet ErrsimModuleSet; + static const int64_t MAX_BUCKET_NUM = 128; + bool is_inited_; + uint64_t tenant_id_; + common::SpinRWLock lock_; + int64_t config_version_; + bool is_whole_module_; + ErrsimModuleSet module_set_; + int64_t percentage_; + DISALLOW_COPY_AND_ASSIGN(ObTenantErrsimModuleMgr); +}; + +} // namespace share +} // namespace oceanbase +#endif diff --git a/src/share/io/ob_io_manager.cpp b/src/share/io/ob_io_manager.cpp index cb52d428f0..5dc54050d1 100644 --- a/src/share/io/ob_io_manager.cpp +++ b/src/share/io/ob_io_manager.cpp @@ -18,6 +18,7 @@ #include "lib/ob_running_mode.h" #include "share/rc/ob_tenant_base.h" #include "logservice/leader_coordinator/ob_failure_detector.h" +#include "share/errsim_module/ob_errsim_module_interface_imp.h" using namespace oceanbase::lib; using namespace oceanbase::common; @@ -314,6 +315,15 @@ int ObIOManager::tenant_aio(const ObIOInfo &info, ObIOHandle &handle) { int ret = OB_SUCCESS; ObRefHolder tenant_holder; +#ifdef ERRSIM + const ObErrsimModuleType type = THIS_WORKER.get_module_type(); + if (is_errsim_module(info.tenant_id_, type.type_)) { + ret = OB_IO_ERROR; + LOG_ERROR("[ERRSIM MODULE] errsim IO error", K(ret), "tenant_id", info.tenant_id_); + return ret; + } +#endif + if (OB_FAIL(get_tenant_io_manager(info.tenant_id_, tenant_holder))) { LOG_WARN("get tenant io manager failed", K(ret), K(info.tenant_id_)); } else if (OB_FAIL(tenant_holder.get_ptr()->inner_aio(info, handle))) { diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index a29f10e360..5e970437f3 100755 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -1570,3 +1570,12 @@ DEF_INT(_ha_tablet_info_batch_count, OB_TENANT_PARAMETER, "0", "[0,]", DEF_TIME(_ha_rpc_timeout, OB_TENANT_PARAMETER, "0", "[0,120s]", "the rpc timeout for storage high availability. Range:[0, 120s]", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + +// for set errsim module types, format like transfer;migration +ERRSIM_DEF_STR_LIST(errsim_module_types, OB_TENANT_PARAMETER, "", + "set module list for errsim error", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +ERRSIM_DEF_DBL(errsim_module_error_percentage, OB_TENANT_PARAMETER, "0", "[0,100]", + "the percentage of the error happened to errsim module. " + "Range: [0, 100] in percentage", + ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)) diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 550ed18255..cd5372b372 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -182,6 +182,8 @@ class ObTenantDagScheduler; class ObTenantModuleInitCtx; class ObGlobalAutoIncService; class ObDagWarningHistoryManager; +class ObTenantErrsimModuleMgr; +class ObTenantErrsimEventMgr; namespace schema { class ObTenantSchemaService; @@ -197,6 +199,14 @@ namespace detector #define ArbMTLMember rootserver::ObArbitrationService*, #endif +#ifdef ERRSIM +#define TenantErrsimModule share::ObTenantErrsimModuleMgr*, +#define TenantErrsimEvent share::ObTenantErrsimEventMgr*, +#else +#define TenantErrsimModule +#define TenantErrsimEvent +#endif + // 在这里列举需要添加的租户局部变量的类型，租户会为每种类型创建一个实例。 // 实例的初始化和销毁逻辑由MTL_BIND接口指定。 // 使用MTL接口可以获取实例。 @@ -290,6 +300,8 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPoolget_module_type()); + THIS_WORKER.set_module_type(type); +#endif THIS_WORKER.set_compatibility_mode(compat_mode); if (OB_FAIL(set_dag_resource(dag->get_consumer_group_id()))) { LOG_WARN("isolate dag CPU and IOPS failed", K(ret)); diff --git a/src/share/scheduler/ob_dag_scheduler.h b/src/share/scheduler/ob_dag_scheduler.h index 4171c50a33..161e9ef3f4 100644 --- a/src/share/scheduler/ob_dag_scheduler.h +++ b/src/share/scheduler/ob_dag_scheduler.h @@ -25,6 +25,7 @@ #include "share/rc/ob_tenant_base.h" #include "share/scheduler/ob_dag_scheduler_config.h" #include "share/ob_table_range.h" +#include "common/errsim_module/ob_errsim_module_type.h" namespace oceanbase { @@ -360,6 +361,10 @@ public: virtual uint64_t get_consumer_group_id() const = 0; int remove_task(ObITask &task); +#ifdef ERRSIM + virtual common::ObErrsimModuleType::TYPE get_module_type() { return ObErrsimModuleType::ERRSIM_MODULE_NONE; } +#endif + protected: int dag_ret_; int64_t add_time_; diff --git a/src/storage/blocksstable/ob_block_manager.cpp b/src/storage/blocksstable/ob_block_manager.cpp index c0a6824a21..f9b4576ebc 100644 --- a/src/storage/blocksstable/ob_block_manager.cpp +++ b/src/storage/blocksstable/ob_block_manager.cpp @@ -32,6 +32,7 @@ #include "storage/ob_super_block_struct.h" #include "storage/slog/ob_storage_logger_manager.h" #include "storage/blocksstable/ob_shared_macro_block_manager.h" +#include "lib/worker.h" using namespace oceanbase::common; using namespace oceanbase::common::hash; @@ -422,6 +423,9 @@ int ObBlockManager::write_super_block(const ObServerSuperBlock &super_block) { int ret = OB_SUCCESS; int64_t write_size = 0; +#ifdef ERRSIM + ErrsimModuleGuard guard(ObErrsimModuleType::ERRSIM_MODULE_NONE); +#endif if (!super_block.is_valid()) { ret = OB_INVALID_ARGUMENT; diff --git a/src/storage/high_availability/ob_rebuild_service.cpp b/src/storage/high_availability/ob_rebuild_service.cpp index 5290815809..a61e3ca650 100644 --- a/src/storage/high_availability/ob_rebuild_service.cpp +++ b/src/storage/high_availability/ob_rebuild_service.cpp @@ -720,14 +720,6 @@ int ObRebuildService::check_can_rebuild_( } else { can_rebuild = true; } - -#ifdef ERRSIM - if (OB_FAIL(ret)) { - //do nothing - } else { - can_rebuild = CHECK_CAN_REBUILD ? false: true; - } -#endif return ret; } diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index 1f994168f0..cb47a466dc 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -32,6 +32,7 @@ #include "storage/tx_storage/ob_ls_service.h" #include "rootserver/ob_tenant_info_loader.h" #include "src/observer/omt/ob_tenant_config.h" +#include "common/errsim_module/ob_errsim_module_type.h" using namespace oceanbase::share; @@ -461,6 +462,25 @@ int64_t ObStorageHAUtils::get_rpc_timeout() return rpc_timeout; } +void ObTransferUtils::set_transfer_module() +{ +#ifdef ERRSIM + if (ObErrsimModuleType::ERRSIM_MODULE_NONE == THIS_WORKER.get_module_type().type_) { + ObErrsimModuleType type(ObErrsimModuleType::ERRSIM_MODULE_TRANSFER); + THIS_WORKER.set_module_type(type); + } +#endif +} + +void ObTransferUtils::clear_transfer_module() +{ +#ifdef ERRSIM + if (ObErrsimModuleType::ERRSIM_MODULE_TRANSFER == THIS_WORKER.get_module_type().type_) { + ObErrsimModuleType type(ObErrsimModuleType::ERRSIM_MODULE_NONE); + THIS_WORKER.set_module_type(type); + } +#endif +} } // end namespace storage } // end namespace oceanbase diff --git a/src/storage/high_availability/ob_storage_ha_utils.h b/src/storage/high_availability/ob_storage_ha_utils.h index 7bf743264a..29c4d736a3 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.h +++ b/src/storage/high_availability/ob_storage_ha_utils.h @@ -62,6 +62,8 @@ struct ObTransferUtils static int kill_tx(const uint64_t tenant_id, const share::ObLSID &ls_id); static int unblock_tx(const uint64_t tenant_id, const share::ObLSID &ls_id); static int get_gts(const uint64_t tenant_id, share::SCN >s); + static void set_transfer_module(); + static void clear_transfer_module(); }; } // end namespace storage diff --git a/src/storage/high_availability/ob_transfer_backfill_tx.h b/src/storage/high_availability/ob_transfer_backfill_tx.h index d3149de2f7..b85183dd42 100644 --- a/src/storage/high_availability/ob_transfer_backfill_tx.h +++ b/src/storage/high_availability/ob_transfer_backfill_tx.h @@ -135,6 +135,9 @@ public: virtual bool operator == (const share::ObIDag &other) const override; virtual int64_t hash() const override; int prepare_ctx(share::ObIDagNet *dag_net); +#ifdef ERRSIM + virtual common::ObErrsimModuleType::TYPE get_module_type() { return ObErrsimModuleType::ERRSIM_MODULE_TRANSFER; } +#endif INHERIT_TO_STRING_KV("ObStorageHADag", ObStorageHADag, KP(this)); private: diff --git a/src/storage/high_availability/ob_transfer_handler.cpp b/src/storage/high_availability/ob_transfer_handler.cpp index 343f9c441f..a7bd0b8218 100644 --- a/src/storage/high_availability/ob_transfer_handler.cpp +++ b/src/storage/high_availability/ob_transfer_handler.cpp @@ -497,16 +497,6 @@ int ObTransferHandler::do_with_start_status_(const share::ObTransferTaskInfo &ta } else { if (OB_FAIL(report_to_meta_table_(task_info))) { LOG_WARN("failed to report to meta table", K(ret), K(task_info)); - } else { -#ifdef ERRSIM - bool is_src_ls_rebuild = EN_MAKE_SRC_LS_REBUILD ? false: true; - ObRebuildService *rebuild_service = MTL(ObRebuildService*); - ObLSRebuildType type(ObLSRebuildType::TRANSFER); - if (!is_src_ls_rebuild) { - } else if (OB_FAIL(rebuild_service->add_rebuild_ls(task_info.src_ls_id_, type))) { - LOG_WARN("failed to add rebuild ls", K(ret), K(task_info)); - } -#endif } } if (OB_SUCCESS != (tmp_ret = record_server_event_(ret, round_, task_info))) { diff --git a/src/storage/high_availability/ob_transfer_service.cpp b/src/storage/high_availability/ob_transfer_service.cpp index 226088bd22..90de6fb9c2 100644 --- a/src/storage/high_availability/ob_transfer_service.cpp +++ b/src/storage/high_availability/ob_transfer_service.cpp @@ -123,6 +123,10 @@ void ObTransferService::run1() { int ret = OB_SUCCESS; lib::set_thread_name("TransferService"); +#ifdef ERRSIM + ObErrsimModuleType module_type(ObErrsimModuleType::ERRSIM_MODULE_TRANSFER); + THIS_WORKER.set_module_type(module_type); +#endif while (!has_set_stop()) { ls_id_array_.reset(); diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp index 5c0380c4bc..45d19042a5 100755 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp @@ -1786,6 +1786,11 @@ int ObTenantMetaMemMgr::compare_and_swap_tablet( const ObMetaDiskAddr &new_addr = new_handle.get_obj()->get_tablet_addr(); const ObTablet *old_tablet = old_handle.get_obj(); const ObTablet *new_tablet = new_handle.get_obj(); + +#ifdef ERRSIM + ErrsimModuleGuard guard(ObErrsimModuleType::ERRSIM_MODULE_NONE); +#endif + if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObTenantMetaMemMgr hasn't been initialized", K(ret)); diff --git a/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp b/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp index f5fbd5047d..ccc8899176 100644 --- a/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_finish_transfer_mds_helper.cpp @@ -280,6 +280,7 @@ int ObTabletFinishTransferOutHelper::on_register( int ret = OB_SUCCESS; ObTXFinishTransferOutInfo tx_finish_transfer_out_info; int64_t pos = 0; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; @@ -294,6 +295,7 @@ int ObTabletFinishTransferOutHelper::on_register( } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(tx_finish_transfer_out_info.src_ls_id_))) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(tx_finish_transfer_out_info)); } + ObTransferUtils::clear_transfer_module(); return ret; } @@ -459,6 +461,7 @@ int ObTabletFinishTransferOutHelper::on_replay( int ret = OB_SUCCESS; ObTXFinishTransferOutInfo tx_finish_transfer_out_info; int64_t pos = 0; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -473,6 +476,7 @@ int ObTabletFinishTransferOutHelper::on_replay( } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_empty_shell_trigger(tx_finish_transfer_out_info.src_ls_id_))) { LOG_WARN("failed to set_tablet_empty_shell_trigger", K(ret), K(tx_finish_transfer_out_info)); } + ObTransferUtils::clear_transfer_module(); return ret; } @@ -859,6 +863,7 @@ int ObTabletFinishTransferInHelper::on_register( ObTXFinishTransferInInfo tx_finish_transfer_in_info; int64_t pos = 0; const bool for_replay = false; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; @@ -871,6 +876,8 @@ int ObTabletFinishTransferInHelper::on_register( } else if (CLICK_FAIL(on_register_success_(tx_finish_transfer_in_info, ctx))) { LOG_WARN("failed to do on register success", K(ret), K(tx_finish_transfer_in_info)); } + + ObTransferUtils::clear_transfer_module(); return ret; } @@ -1072,6 +1079,7 @@ int ObTabletFinishTransferInHelper::on_replay( int64_t pos = 0; bool skip_replay = false; const bool for_replay = true; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -1084,6 +1092,7 @@ int ObTabletFinishTransferInHelper::on_replay( } else if (CLICK_FAIL(on_replay_success_(scn, tx_finish_transfer_in_info, ctx))) { LOG_WARN("failed to do on_replay_success_", K(ret), K(tx_finish_transfer_in_info)); } + ObTransferUtils::clear_transfer_module(); return ret; } diff --git a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp index 11d01a0974..05480036ad 100644 --- a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp @@ -191,6 +191,7 @@ int ObTabletStartTransferOutHelper::on_register( ObTXStartTransferOutInfo tx_start_transfer_out_info; int64_t pos = 0; const bool for_replay = false; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; @@ -203,6 +204,7 @@ int ObTabletStartTransferOutHelper::on_register( } else if (CLICK_FAIL(on_register_success_(tx_start_transfer_out_info, ctx))) { LOG_WARN("failed to on register", K(ret), K(tx_start_transfer_out_info)); } + ObTransferUtils::clear_transfer_module(); return ret; } @@ -410,6 +412,7 @@ int ObTabletStartTransferOutHelper::on_replay( ObTXStartTransferOutInfo tx_start_transfer_out_info; int64_t pos = 0; const bool for_replay = true; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -441,6 +444,7 @@ int ObTabletStartTransferOutHelper::on_replay( "scn", scn); #endif DEBUG_SYNC(AFTER_ON_REDO_START_TRANSFER_OUT); + ObTransferUtils::clear_transfer_module(); return ret; } @@ -680,6 +684,7 @@ int ObTabletStartTransferInHelper::on_register( ObTXStartTransferInInfo tx_start_transfer_in_info; int64_t pos = 0; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; LOG_WARN("on register start transfer in get invalid argument", K(ret), KP(buf), K(len)); @@ -693,6 +698,7 @@ int ObTabletStartTransferInHelper::on_register( } else if (CLICK_FAIL(ObTabletCreateDeleteMdsUserData::set_tablet_gc_trigger(tx_start_transfer_in_info.dest_ls_id_))) { LOG_WARN("failed to set_tablet_gc_trigger", K(ret), K(tx_start_transfer_in_info)); } + ObTransferUtils::clear_transfer_module(); return ret; } @@ -1215,6 +1221,7 @@ int ObTabletStartTransferInHelper::on_replay( int64_t pos = 0; bool skip_replay = false; ObTransferService *transfer_service = nullptr; + ObTransferUtils::set_transfer_module(); if (OB_ISNULL(buf) || len < 0) { ret = OB_INVALID_ARGUMENT; @@ -1242,6 +1249,7 @@ int ObTabletStartTransferInHelper::on_replay( SERVER_EVENT_SYNC_ADD("TRANSFER", "AFTER_ON_REDO_START_TRANSFER_IN"); #endif DEBUG_SYNC(AFTER_ON_REDO_START_TRANSFER_IN); + ObTransferUtils::clear_transfer_module(); return ret; } @@ -1568,6 +1576,8 @@ bool ObTabletStartTransferInHelper::check_can_replay_commit( bool skip_replay = false; ObTransferService *transfer_service = nullptr; bool can_skip_check_src = false; + ObTransferUtils::set_transfer_module(); + LOG_INFO("check can replay start transfer in commit", K(scn)); if (OB_ISNULL(buf) || len < 0 || !scn.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -1599,6 +1609,7 @@ bool ObTabletStartTransferInHelper::check_can_replay_commit( transfer_service->wakeup(); } } + ObTransferUtils::clear_transfer_module(); return b_ret; }