/** * Copyright (c) 2023 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: * http://license.coscl.org.cn/MulanPubL-2.0 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ #define USING_LOG_PREFIX SHARE #include "lib/random/ob_random.h" #include "share/ob_ddl_sim_point.h" #include "share/config/ob_server_config.h" using namespace oceanbase::common; using namespace oceanbase::share; int64_t ObTenantDDLSimContext::to_string(char* buf, const int64_t buf_len) const { int64_t pos = 0; J_OBJ_START(); const int64_t MAX_FIXED_POINT_COUNT = 64; ObDDLSimPointID fixed_points[MAX_FIXED_POINT_COUNT]; int fixed_point_count = 0; for (int64_t i = 0; i < MAX_DDL_SIM_POINT_ID; ++i) { if (nullptr != fixed_points_ && fixed_points_[i] && i < MAX_FIXED_POINT_COUNT) { fixed_points[fixed_point_count++] = static_cast(i); } } J_KV(K(tenant_id_), K(type_), K(seed_), K(trigger_percent_), "fixed_points_", ObArrayWrap(fixed_points, fixed_point_count)); J_OBJ_END(); return pos; } ObDDLSimPointMgr &ObDDLSimPointMgr::get_instance() { static ObDDLSimPointMgr instance; return instance; } ObDDLSimPointMgr::ObDDLSimPointMgr() : is_inited_(false), arena_("ddl_sim_pnt_mgr") { memset(all_points_, 0, sizeof(all_points_)); } int ObDDLSimPointMgr::init() { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("ddl sim point mgr already inited", K(ret), K(is_inited_)); } else if (OB_FAIL(tenant_map_.create(207, "ddl_sim_tnt_map"))) { LOG_WARN("create tenant context map failed"); } else if (OB_FAIL(task_sim_map_.create(199999, "ddl_sim_pnt_map"))) { LOG_WARN("create task sim map failed", K(ret)); } else { // 1. remember sim action size #define RET_ERR(args...) sizeof(ObDDLSimRetAction) #define N_RET_ERR(max_repeat_times, args...) sizeof(ObDDLSimRetAction) #define SLEEP_MS(args...) sizeof(ObDDLSimSleepAction) #define N_SLEEP_MS(args...) sizeof(ObDDLSimSleepAction) #define DDL_SIM_POINT_DEFINE(type, name, id, desc, action_size) all_points_[id].action_size_ = action_size; #include "share/ob_ddl_sim_point_define.h" #undef DDL_SIM_POINT_DEFINE #undef RET_ERR #undef N_RET_ERR #undef SLEEP_MS #undef N_SLEEP_MS // 2. constuct sim point void *buf = nullptr; #define RET_ERR(ret_code, args...) new (buf) ObDDLSimRetAction<1 + ARGS_NUM(args)>(1, {ret_code, ##args}) #define N_RET_ERR(max_repeat_times, ret_code, args...) new (buf) ObDDLSimRetAction<1 + ARGS_NUM(args)>(max_repeat_times, {ret_code, ##args}) #define SLEEP_MS(min_time, max_time...) new (buf) ObDDLSimSleepAction(1, min_time, ##max_time) #define N_SLEEP_MS(max_repeat_times, min_time, max_time...) new (buf) ObDDLSimSleepAction(max_repeat_times, min_time, ##max_time) #define DDL_SIM_POINT_DEFINE(type, name, id, desc, action) \ if (OB_SUCC(ret)) {\ if (OB_ISNULL(buf = ob_malloc(all_points_[id].action_size_, "ddl_sim_act"))) {\ ret = OB_ALLOCATE_MEMORY_FAILED;\ LOG_WARN("allocate memory for ddl sim action failed", K(ret), K(all_points_[id].action_size_));\ } else {\ all_points_[id] = ObDDLSimPoint(name, type, #name, desc, action);\ }\ } #include "share/ob_ddl_sim_point_define.h" #undef DDL_SIM_POINT_DEFINE #undef RET_ERR #undef N_RET_ERR #undef SLEEP_MS #undef N_SLEEP_MS } if (OB_SUCC(ret)) { is_inited_ = true; } return ret; } class TenantContextUpdater { public: TenantContextUpdater(const ObTenantDDLSimContext &tenant_context, const ObIArray &fixed_points_array) : new_context_(tenant_context), fixed_point_array_(fixed_points_array) {} ~TenantContextUpdater() = default; int operator() (hash::HashMapPair &entry) { int ret = OB_SUCCESS; if (new_context_.trigger_percent_ > 0 && 0 == entry.second.trigger_percent_) { entry.second.seed_ = new_context_.seed_; entry.second.trigger_percent_ = new_context_.trigger_percent_; entry.second.type_ = new_context_.type_; } const int64_t point_map_size = sizeof(bool) * MAX_DDL_SIM_POINT_ID; if (fixed_point_array_.count() > 0) { if (nullptr == entry.second.fixed_points_) { void *buf = ObDDLSimPointMgr::get_instance().get_arena_allocator().alloc(point_map_size); if (OB_ISNULL(buf)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret), K(point_map_size)); } else { bool *tmp_map = new bool[MAX_DDL_SIM_POINT_ID]; memset(tmp_map, 0, point_map_size); entry.second.fixed_points_ = tmp_map; } } } if (nullptr != entry.second.fixed_points_) { memset(entry.second.fixed_points_, 0, point_map_size); } for (int64_t i = 0; OB_SUCC(ret) && i < fixed_point_array_.count(); ++i) { const int64_t point_id = fixed_point_array_.at(i); entry.second.fixed_points_[point_id] = true; } LOG_INFO("update tenant param of ddl sim point success", K(new_context_), K(fixed_point_array_), K(entry.second)); return OB_SUCCESS; } public: const ObTenantDDLSimContext &new_context_; const ObIArray &fixed_point_array_; }; int ObDDLSimPointMgr::set_tenant_param(const uint64_t tenant_id, const ObConfigIntListItem &rand_param, const ObConfigIntListItem &fixed_param) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(tenant_id)); } else { ObArray fixed_point_array; ObTenantDDLSimContext tenant_context; tenant_context.tenant_id_ = tenant_id; // 1. fill random param if need if (rand_param.size() >= 2) { tenant_context.seed_ = rand_param[0]; tenant_context.trigger_percent_ = rand_param[1]; if (rand_param.size() >= 3) { tenant_context.type_ = static_cast(rand_param[2]); } } // 2. try push tenant context into tenant map if (OB_FAIL(tenant_map_.set_refactored(tenant_id, tenant_context))) { if (OB_HASH_EXIST != ret) { LOG_WARN("set tenant context failed", K(ret)); } else { ret = OB_SUCCESS; } } // 3. fill fixed param if need for (int i = 0; OB_SUCC(ret) && i < fixed_param.size(); ++i) { const int64_t point_id = fixed_param[i]; if (point_id <= MIN_DDL_SIM_POINT_ID || point_id >= MAX_DDL_SIM_POINT_ID) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(point_id)); ret = OB_SUCCESS; // ignore invalid point id } else if (!all_points_[point_id].is_valid()) { // do nothing } else if (OB_FAIL(fixed_point_array.push_back(point_id))) { LOG_WARN("push back fixed point failed", K(ret), K(point_id), K(i)); } } // 4. update tenant context if (OB_SUCC(ret)) { TenantContextUpdater updater(tenant_context, fixed_point_array); if (OB_FAIL(tenant_map_.atomic_refactored(tenant_id, updater))) { LOG_WARN("update tenant context failed", K(ret), K(tenant_id), K(tenant_context)); } } } return ret; } int ObDDLSimPointMgr::generate_task_sim_map(const ObTenantDDLSimContext &tenant_context, const int64_t current_task_id, const std::initializer_list &point_ids) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); } else if (OB_UNLIKELY(tenant_context.trigger_percent_ <= 0)) { // skip } else { const uint64_t tenant_id = tenant_context.tenant_id_; const int64_t seed = tenant_context.seed_; const int64_t trigger_percent = tenant_context.trigger_percent_; for (std::initializer_list::iterator it = point_ids.begin(); OB_SUCC(ret) && it != point_ids.end(); ++it) { ObDDLSimPointID point_id = *it; const ObDDLSimPoint &cur_sim_point = all_points_[point_id]; if (cur_sim_point.is_valid() && (SIM_TYPE_ALL == tenant_context.type_ || tenant_context.type_ == cur_sim_point.type_)) { srand(static_cast((tenant_id + seed) * current_task_id * point_id)); if (rand() % 100 < trigger_percent) { if (OB_FAIL(task_sim_map_.set_refactored(TaskSimPoint(tenant_id, current_task_id, point_id), 0))) { if (OB_HASH_EXIST != ret) { LOG_WARN("set task sim point into map failed", K(ret), K(tenant_id), K(current_task_id), K(point_id)); } else { ret = OB_SUCCESS; } } } } } } return ret; } class SimCountUpdater { public: explicit SimCountUpdater(int64_t step) : step_(step), old_trigger_count_(0) {} ~SimCountUpdater() = default; int operator() (hash::HashMapPair &entry) { old_trigger_count_ = entry.second; entry.second += step_; return OB_SUCCESS; } public: int64_t step_; int64_t old_trigger_count_; }; int ObDDLSimPointMgr::try_sim(const uint64_t tenant_id, const uint64_t task_id, const std::initializer_list &point_ids) { int ret = OB_SUCCESS; ObTenantDDLSimContext tenant_context; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || task_id < 0)) { LOG_INFO("invalid argument for ddl errsim, ignore", K(tenant_id), K(task_id)); } else if (0 == task_id) { // task_id maybe not set, skip } else if (OB_FAIL(tenant_map_.get_refactored(tenant_id, tenant_context))) { if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("get tenant context failed", K(ret), K(tenant_id)); } else { ret = OB_SUCCESS; } } else if (OB_FAIL(generate_task_sim_map(tenant_context, task_id, point_ids))) { LOG_WARN("generate task sim map failed", K(ret), K(tenant_context), K(task_id)); } else { for (std::initializer_list::iterator it = point_ids.begin(); OB_SUCC(ret) && it != point_ids.end(); ++it) { ObDDLSimPointID point_id = *it; bool need_execute = false; TaskSimPoint sim_key(tenant_id, task_id, point_id); if (point_id <= MIN_DDL_SIM_POINT_ID || point_id > MAX_DDL_SIM_POINT_ID || !all_points_[point_id].is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid point id", K(ret), K(point_id)); } else if (nullptr != tenant_context.fixed_points_ && tenant_context.fixed_points_[point_id]) { int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(task_sim_map_.set_refactored(TaskSimPoint(tenant_id, task_id, point_id), 0))) { if (OB_HASH_EXIST != tmp_ret) { LOG_WARN("set fixed point into task sim map failed", K(tmp_ret), K(tenant_id), K(task_id), K(point_id)); } } } if (OB_SUCC(ret)) { SimCountUpdater inc(1); if (OB_FAIL(task_sim_map_.atomic_refactored(sim_key, inc))) { if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("check task need sim ddl point failed", K(ret), K(sim_key)); } else { ret = OB_SUCCESS; } } else { need_execute = inc.old_trigger_count_ < all_points_[sim_key.point_id_].action_->max_repeat_times_; if (need_execute) { ret = all_points_[point_id].action_->execute(); LOG_INFO("ddl sim point executed", K(ret), K(sim_key)); } else { SimCountUpdater dec(-1); int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(task_sim_map_.atomic_refactored(sim_key, dec))) { LOG_WARN("decrease trigger count failed", K(tmp_ret), K(sim_key)); } } } } } } return ret; } int ObDDLSimPointMgr::get_sim_point(const int64_t idx, ObDDLSimPoint &sim_point) const { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); } else if (OB_UNLIKELY(idx < MIN_DDL_SIM_POINT_ID || idx >= MAX_DDL_SIM_POINT_ID)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(idx)); } else { sim_point = all_points_[idx]; } return ret; } class SimCountCollector { public: SimCountCollector(ObIArray &task_sim_points, ObIArray &sim_counts) : task_sim_points_(task_sim_points), sim_counts_(sim_counts) {} ~SimCountCollector() = default; int operator() (hash::HashMapPair &entry) { int ret = OB_SUCCESS; if (OB_FAIL(task_sim_points_.push_back(entry.first))) { LOG_WARN("push back task sim point failed", K(ret), K(entry.first)); } else if (OB_FAIL(sim_counts_.push_back(entry.second))) { LOG_WARN("push back sim count failed", K(ret), K(entry.second)); } return ret; } public: ObIArray &task_sim_points_; ObIArray &sim_counts_; }; int ObDDLSimPointMgr::get_sim_stat(ObIArray &task_sim_points, ObIArray &sim_counts) { int ret = OB_SUCCESS; task_sim_points.reset(); sim_counts.reset(); int64_t entry_count = 0; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(is_inited_)); } else { entry_count = task_sim_map_.size(); } if (OB_SUCC(ret) && entry_count > 0) { const int64_t reserve_count = entry_count + 1000L; if (OB_FAIL(task_sim_points.reserve(reserve_count))) { LOG_WARN("reserve array capacity failed", K(ret), K(entry_count), K(reserve_count)); } else if (OB_FAIL(sim_counts.reserve(reserve_count))) { LOG_WARN("reserve array capacity failed", K(ret), K(entry_count), K(reserve_count)); } } if (OB_SUCC(ret) && entry_count > 0) { SimCountCollector stat_collector(task_sim_points, sim_counts); if (OB_FAIL(task_sim_map_.foreach_refactored(stat_collector))) { LOG_WARN("collect ddl sim entry failed", K(ret)); } } return ret; }