hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] enhance_ddl_quality

Browse Source 
Co-authored-by: renju96 <fcbrenju@163.com>
This commit is contained in:
simonjoylet
2023-11-23 04:11:38 +00:00
committed by ob-robot
parent fd1115b382
commit 16f37c94b6
74 changed files with 3169 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

368
src/share/ob_ddl_sim_point.cpp Normal file
View File

@ -0,0 +1,368 @@
/**
* Copyright (c) 2023 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SHARE
#include "lib/random/ob_random.h"
#include "share/ob_ddl_sim_point.h"
#include "share/config/ob_server_config.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
int64_t ObTenantDDLSimContext::to_string(char* buf, const int64_t buf_len) const
{
int64_t pos = 0;
J_OBJ_START();
const int64_t MAX_FIXED_POINT_COUNT = 64;
ObDDLSimPointID fixed_points[MAX_FIXED_POINT_COUNT];
int fixed_point_count = 0;
for (int64_t i = 0; i < MAX_DDL_SIM_POINT_ID; ++i) {
if (nullptr != fixed_points_ && fixed_points_[i] && i < MAX_FIXED_POINT_COUNT) {
fixed_points[fixed_point_count++] = static_cast<ObDDLSimPointID>(i);
}
}
J_KV(K(tenant_id_), K(type_), K(seed_), K(trigger_percent_),
"fixed_points_", ObArrayWrap<ObDDLSimPointID>(fixed_points, fixed_point_count));
J_OBJ_END();
return pos;
}
ObDDLSimPointMgr &ObDDLSimPointMgr::get_instance()
{
static ObDDLSimPointMgr instance;
return instance;
}
ObDDLSimPointMgr::ObDDLSimPointMgr()
: is_inited_(false), arena_("ddl_sim_pnt_mgr")
{
memset(all_points_, 0, sizeof(all_points_));
}
int ObDDLSimPointMgr::init()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ddl sim point mgr already inited", K(ret), K(is_inited_));
} else if (OB_FAIL(tenant_map_.create(207, "ddl_sim_tnt_map"))) {
LOG_WARN("create tenant context map failed");
} else if (OB_FAIL(task_sim_map_.create(199999, "ddl_sim_pnt_map"))) {
LOG_WARN("create task sim map failed", K(ret));
} else {
// 1. remember sim action size
#define RET_ERR(args...) sizeof(ObDDLSimRetAction<ARGS_NUM(args)>)
#define N_RET_ERR(max_repeat_times, args...) sizeof(ObDDLSimRetAction<ARGS_NUM(args)>)
#define SLEEP_MS(args...) sizeof(ObDDLSimSleepAction)
#define N_SLEEP_MS(args...) sizeof(ObDDLSimSleepAction)
#define DDL_SIM_POINT_DEFINE(type, name, id, desc, action_size) all_points_[id].action_size_ = action_size;
#include "share/ob_ddl_sim_point_define.h"
#undef DDL_SIM_POINT_DEFINE
#undef RET_ERR
#undef N_RET_ERR
#undef SLEEP_MS
#undef N_SLEEP_MS
// 2. constuct sim point
void *buf = nullptr;
#define RET_ERR(ret_code, args...) new (buf) ObDDLSimRetAction<1 + ARGS_NUM(args)>(1, {ret_code, ##args})
#define N_RET_ERR(max_repeat_times, ret_code, args...) new (buf) ObDDLSimRetAction<1 + ARGS_NUM(args)>(max_repeat_times, {ret_code, ##args})
#define SLEEP_MS(min_time, max_time...) new (buf) ObDDLSimSleepAction(1, min_time, ##max_time)
#define N_SLEEP_MS(max_repeat_times, min_time, max_time...) new (buf) ObDDLSimSleepAction(max_repeat_times, min_time, ##max_time)
#define DDL_SIM_POINT_DEFINE(type, name, id, desc, action) \
if (OB_SUCC(ret)) {\
if (OB_ISNULL(buf = ob_malloc(all_points_[id].action_size_, "ddl_sim_act"))) {\
ret = OB_ALLOCATE_MEMORY_FAILED;\
LOG_WARN("allocate memory for ddl sim action failed", K(ret), K(all_points_[id].action_size_));\
} else {\
all_points_[id] = ObDDLSimPoint(name, type, #name, desc, action);\
}\
}
#include "share/ob_ddl_sim_point_define.h"
#undef DDL_SIM_POINT_DEFINE
#undef RET_ERR
#undef N_RET_ERR
#undef SLEEP_MS
#undef N_SLEEP_MS
}
if (OB_SUCC(ret)) {
is_inited_ = true;
}
return ret;
}
class TenantContextUpdater
{
public:
TenantContextUpdater(const ObTenantDDLSimContext &tenant_context, const ObIArray<int64_t> &fixed_points_array)
: new_context_(tenant_context), fixed_point_array_(fixed_points_array) {}
~TenantContextUpdater() = default;
int operator() (hash::HashMapPair<uint64_t, ObTenantDDLSimContext> &entry) {
int ret = OB_SUCCESS;
if (new_context_.trigger_percent_ > 0 && 0 == entry.second.trigger_percent_) {
entry.second.seed_ = new_context_.seed_;
entry.second.trigger_percent_ = new_context_.trigger_percent_;
entry.second.type_ = new_context_.type_;
}
const int64_t point_map_size = sizeof(bool) * MAX_DDL_SIM_POINT_ID;
if (fixed_point_array_.count() > 0) {
if (nullptr == entry.second.fixed_points_) {
void *buf = ObDDLSimPointMgr::get_instance().get_arena_allocator().alloc(point_map_size);
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret), K(point_map_size));
} else {
bool *tmp_map = new bool[MAX_DDL_SIM_POINT_ID];
memset(tmp_map, 0, point_map_size);
entry.second.fixed_points_ = tmp_map;
}
}
}
if (nullptr != entry.second.fixed_points_) {
memset(entry.second.fixed_points_, 0, point_map_size);
}
for (int64_t i = 0; OB_SUCC(ret) && i < fixed_point_array_.count(); ++i) {
const int64_t point_id = fixed_point_array_.at(i);
entry.second.fixed_points_[point_id] = true;
}
LOG_INFO("update tenant param of ddl sim point success", K(new_context_), K(fixed_point_array_), K(entry.second));
return OB_SUCCESS;
}
public:
const ObTenantDDLSimContext &new_context_;
const ObIArray<int64_t> &fixed_point_array_;
};
int ObDDLSimPointMgr::set_tenant_param(const uint64_t tenant_id, const ObConfigIntListItem &rand_param, const ObConfigIntListItem &fixed_param)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(tenant_id));
} else {
ObArray<int64_t> fixed_point_array;
ObTenantDDLSimContext tenant_context;
tenant_context.tenant_id_ = tenant_id;
// 1. fill random param if need
if (rand_param.size() >= 2) {
tenant_context.seed_ = rand_param[0];
tenant_context.trigger_percent_ = rand_param[1];
if (rand_param.size() >= 3) {
tenant_context.type_ = static_cast<ObSimType>(rand_param[2]);
}
}
// 2. try push tenant context into tenant map
if (OB_FAIL(tenant_map_.set_refactored(tenant_id, tenant_context))) {
if (OB_HASH_EXIST != ret) {
LOG_WARN("set tenant context failed", K(ret));
} else {
ret = OB_SUCCESS;
}
}
// 3. fill fixed param if need
for (int i = 0; OB_SUCC(ret) && i < fixed_param.size(); ++i) {
const int64_t point_id = fixed_param[i];
if (point_id <= MIN_DDL_SIM_POINT_ID || point_id >= MAX_DDL_SIM_POINT_ID) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(point_id));
ret = OB_SUCCESS; // ignore invalid point id
} else if (!all_points_[point_id].is_valid()) {
// do nothing
} else if (OB_FAIL(fixed_point_array.push_back(point_id))) {
LOG_WARN("push back fixed point failed", K(ret), K(point_id), K(i));
}
}
// 4. update tenant context
if (OB_SUCC(ret)) {
TenantContextUpdater updater(tenant_context, fixed_point_array);
if (OB_FAIL(tenant_map_.atomic_refactored(tenant_id, updater))) {
LOG_WARN("update tenant context failed", K(ret), K(tenant_id), K(tenant_context));
}
}
}
return ret;
}
int ObDDLSimPointMgr::generate_task_sim_map(const ObTenantDDLSimContext &tenant_context, const int64_t current_task_id, const std::initializer_list<ObDDLSimPointID> &point_ids)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(tenant_context.trigger_percent_ <= 0)) {
// skip
} else {
const uint64_t tenant_id = tenant_context.tenant_id_;
const int64_t seed = tenant_context.seed_;
const int64_t trigger_percent = tenant_context.trigger_percent_;
for (std::initializer_list<ObDDLSimPointID>::iterator it = point_ids.begin(); OB_SUCC(ret) && it != point_ids.end(); ++it) {
ObDDLSimPointID point_id = *it;
const ObDDLSimPoint &cur_sim_point = all_points_[point_id];
if (cur_sim_point.is_valid() && (SIM_TYPE_ALL == tenant_context.type_ || tenant_context.type_ == cur_sim_point.type_)) {
srand(static_cast<int32_t>((tenant_id + seed) * current_task_id * point_id));
if (rand() % 100 < trigger_percent) {
if (OB_FAIL(task_sim_map_.set_refactored(TaskSimPoint(tenant_id, current_task_id, point_id), 0))) {
if (OB_HASH_EXIST != ret) {
LOG_WARN("set task sim point into map failed", K(ret), K(tenant_id), K(current_task_id), K(point_id));
} else {
ret = OB_SUCCESS;
}
}
}
}
}
}
return ret;
}
class SimCountUpdater
{
public:
explicit SimCountUpdater(int64_t step) : step_(step), old_trigger_count_(0) {}
~SimCountUpdater() = default;
int operator() (hash::HashMapPair<ObDDLSimPointMgr::TaskSimPoint, int64_t> &entry) {
old_trigger_count_ = entry.second;
entry.second += step_;
return OB_SUCCESS;
}
public:
int64_t step_;
int64_t old_trigger_count_;
};
int ObDDLSimPointMgr::try_sim(const uint64_t tenant_id, const uint64_t task_id, const std::initializer_list<ObDDLSimPointID> &point_ids)
{
int ret = OB_SUCCESS;
ObTenantDDLSimContext tenant_context;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || task_id < 0)) {
LOG_INFO("invalid argument for ddl errsim, ignore", K(tenant_id), K(task_id));
} else if (0 == task_id) {
// task_id maybe not set, skip
} else if (OB_FAIL(tenant_map_.get_refactored(tenant_id, tenant_context))) {
if (OB_HASH_NOT_EXIST != ret) {
LOG_WARN("get tenant context failed", K(ret), K(tenant_id));
} else {
ret = OB_SUCCESS;
}
} else if (OB_FAIL(generate_task_sim_map(tenant_context, task_id, point_ids))) {
LOG_WARN("generate task sim map failed", K(ret), K(tenant_context), K(task_id));
} else {
for (std::initializer_list<ObDDLSimPointID>::iterator it = point_ids.begin(); OB_SUCC(ret) && it != point_ids.end(); ++it) {
ObDDLSimPointID point_id = *it;
bool need_execute = false;
TaskSimPoint sim_key(tenant_id, task_id, point_id);
if (point_id <= MIN_DDL_SIM_POINT_ID || point_id > MAX_DDL_SIM_POINT_ID || !all_points_[point_id].is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid point id", K(ret), K(point_id));
} else if (nullptr != tenant_context.fixed_points_ && tenant_context.fixed_points_[point_id]) {
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(task_sim_map_.set_refactored(TaskSimPoint(tenant_id, task_id, point_id), 0))) {
if (OB_HASH_EXIST != tmp_ret) {
LOG_WARN("set fixed point into task sim map failed", K(tmp_ret), K(tenant_id), K(task_id), K(point_id));
}
}
}
if (OB_SUCC(ret)) {
SimCountUpdater inc(1);
if (OB_FAIL(task_sim_map_.atomic_refactored(sim_key, inc))) {
if (OB_HASH_NOT_EXIST != ret) {
LOG_WARN("check task need sim ddl point failed", K(ret), K(sim_key));
} else {
ret = OB_SUCCESS;
}
} else {
need_execute = inc.old_trigger_count_ < all_points_[sim_key.point_id_].action_->max_repeat_times_;
if (need_execute) {
ret = all_points_[point_id].action_->execute();
LOG_INFO("ddl sim point executed", K(ret), K(sim_key));
} else {
SimCountUpdater dec(-1);
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(task_sim_map_.atomic_refactored(sim_key, dec))) {
LOG_WARN("decrease trigger count failed", K(tmp_ret), K(sim_key));
}
}
}
}
}
}
return ret;
}
int ObDDLSimPointMgr::get_sim_point(const int64_t idx, ObDDLSimPoint &sim_point) const
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(idx < MIN_DDL_SIM_POINT_ID || idx >= MAX_DDL_SIM_POINT_ID)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(idx));
} else {
sim_point = all_points_[idx];
}
return ret;
}
class SimCountCollector
{
public:
SimCountCollector(ObIArray<ObDDLSimPointMgr::TaskSimPoint> &task_sim_points, ObIArray<int64_t> &sim_counts)
: task_sim_points_(task_sim_points), sim_counts_(sim_counts) {}
~SimCountCollector() = default;
int operator() (hash::HashMapPair<ObDDLSimPointMgr::TaskSimPoint, int64_t> &entry) {
int ret = OB_SUCCESS;
if (OB_FAIL(task_sim_points_.push_back(entry.first))) {
LOG_WARN("push back task sim point failed", K(ret), K(entry.first));
} else if (OB_FAIL(sim_counts_.push_back(entry.second))) {
LOG_WARN("push back sim count failed", K(ret), K(entry.second));
}
return ret;
}
public:
ObIArray<ObDDLSimPointMgr::TaskSimPoint> &task_sim_points_;
ObIArray<int64_t> &sim_counts_;
};
int ObDDLSimPointMgr::get_sim_stat(ObIArray<TaskSimPoint> &task_sim_points, ObIArray<int64_t> &sim_counts)
{
int ret = OB_SUCCESS;
task_sim_points.reset();
sim_counts.reset();
int64_t entry_count = 0;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else {
entry_count = task_sim_map_.size();
}
if (OB_SUCC(ret) && entry_count > 0) {
const int64_t reserve_count = entry_count + 1000L;
if (OB_FAIL(task_sim_points.reserve(reserve_count))) {
LOG_WARN("reserve array capacity failed", K(ret), K(entry_count), K(reserve_count));
} else if (OB_FAIL(sim_counts.reserve(reserve_count))) {
LOG_WARN("reserve array capacity failed", K(ret), K(entry_count), K(reserve_count));
}
}
if (OB_SUCC(ret) && entry_count > 0) {
SimCountCollector stat_collector(task_sim_points, sim_counts);
if (OB_FAIL(task_sim_map_.foreach_refactored(stat_collector))) {
LOG_WARN("collect ddl sim entry failed", K(ret));
}
}
return ret;
}