hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix resource manager refresh directive, group, plan

Browse Source
This commit is contained in:
zhjc1124 2024-11-07 13:14:01 +00:00 committed by ob-robot
parent 0ffe32ffe0
commit 886b2b1e5e
12 changed files with 296 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/observer/omt/ob_tenant.cpp
View File

@ -1153,11 +1153,6 @@ void ObTenant::destroy()
DESTROY_ENTITY(ctx_);
ctx_ = nullptr;
}
if (cgroup_ctrl_.is_valid() &&
OB_TMP_FAIL(cgroup_ctrl_.remove_both_cgroup(
id_, OB_INVALID_GROUP_ID, GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) {
LOG_WARN_RET(tmp_ret, "remove tenant cgroup failed", K(tmp_ret), K_(id));
}
group_map_.destroy_group();
ObTenantSwitchGuard guard(this);
print_all_thread("TENANT_BEFORE_DESTROY", id_);
@ -1176,6 +1171,11 @@ void ObTenant::destroy()
common::ob_delete(mtl_init_ctx_);
mtl_init_ctx_ = nullptr;
}
if (cgroup_ctrl_.is_valid() &&
OB_TMP_FAIL(cgroup_ctrl_.remove_both_cgroup(
id_, OB_INVALID_GROUP_ID, GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) {
LOG_WARN_RET(tmp_ret, "remove tenant cgroup failed", K(tmp_ret), K_(id));
}
}
void ObTenant::set_unit_max_cpu(double cpu)
@ -1648,8 +1648,7 @@ void ObTenant::print_throttled_time()
const uint64_t MODE_CNT = static_cast<uint64_t>(ObIOMode::MAX_MODE) + 1;
for (int64_t i = 0; i < tenant_holder.get_ptr()->get_group_num(); i++) {
uint64_t group_config_index = i * MODE_CNT;
if (!tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).deleted_ &&
!tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).cleared_) {
if (!tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).deleted_) {
uint64_t group_id = tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).group_id_;
if (OB_TMP_FAIL(tenant_holder.get_ptr()->get_throttled_time(group_id, group_throttled_time))) {
LOG_WARN_RET(tmp_ret, "get throttled time failed", K(tmp_ret), K(group_id));

5
src/observer/virtual_table/ob_all_virtual_io_status.cpp
View File

@ -395,8 +395,7 @@ int ObAllVirtualIOQuota::record_user_group(const uint64_t tenant_id, ObIOUsage &
for (int64_t i = 0; i < info.count(); ++i) {
if (OB_TMP_FAIL(oceanbase::common::transform_usage_index_to_group_config_index(i, group_config_index))) {
} else if (group_config_index >= io_config.group_configs_.count()) {
} else if (io_config.group_configs_.at(group_config_index).deleted_ ||
io_config.group_configs_.at(group_config_index).cleared_) {
} else if (io_config.group_configs_.at(group_config_index).deleted_) {
} else if (info.at(i).avg_byte_ > std::numeric_limits<double>::epsilon()) {
QuotaInfo item;
ObIOMode mode = static_cast<ObIOMode>(group_config_index % MODE_COUNT);
@ -676,7 +675,7 @@ int ObAllVirtualIOScheduler::init(const common::ObAddr &addr)
const ObTenantIOConfig &io_config = tenant_holder.get_ptr()->get_io_config();
int64_t group_num = io_config.group_configs_.count();
for (int64_t index = 0; OB_SUCC(ret) && index < group_num; ++index) {
if (io_config.group_configs_.at(index).deleted_ || io_config.group_configs_.at(index).cleared_) {
if (io_config.group_configs_.at(index).deleted_) {
continue;
}
ScheduleInfo item;

2
src/pl/sys_package/ob_pl_dbms_resource_manager.cpp
View File

@ -92,7 +92,7 @@ int ObPlDBMSResourceManager::delete_plan(
}
if (OB_SUCC(ret)) {
if (OB_FAIL(proxy.delete_plan(tenant_id, plan))) {
LOG_WARN("fail create plan", K(tenant_id), K(plan), K(ret));
LOG_WARN("fail delete plan", K(tenant_id), K(plan), K(ret));
}
}
return ret;

116
src/share/resource_manager/ob_cgroup_ctrl.cpp
View File

@ -801,122 +801,6 @@ int ObCgroupCtrl::get_throttled_time(const uint64_t tenant_id, int64_t &throttle
return ret;
}
int ObCgroupCtrl::set_group_iops(const uint64_t tenant_id,
const int64_t group_id,
const ObGroupIOInfo &group_io)
{
int ret = OB_SUCCESS;
ObRefHolder<ObTenantIOManager> tenant_holder;
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_CONFIG;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant_holder.get_ptr()->modify_io_config(group_id,
group_io.group_name_,
group_io.min_percent_,
group_io.max_percent_,
group_io.weight_percent_,
group_io.max_net_bandwidth_percent_,
group_io.net_bandwidth_weight_percent_))) {
LOG_WARN("modify consumer group iops failed", K(ret), K(group_id), K(tenant_id), K(group_id));
}
return ret;
}
int ObCgroupCtrl::reset_all_group_iops(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
ObRefHolder<ObTenantIOManager> tenant_holder;
// 删除plan, IO层代表对应的所有group资源为0,0,0, 但group对应的数据结构不会被释放以防用户后续复用
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_CONFIG;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant_holder.get_ptr()->reset_all_group_config())) {
LOG_WARN("reset consumer group iops failed", K(ret), K(tenant_id));
} else {
LOG_INFO("group iops control has reset, delete cur plan success", K(tenant_id));
}
return ret;
}
int ObCgroupCtrl::reset_group_iops(const uint64_t tenant_id,
const common::ObString &consumer_group)
{
int ret = OB_SUCCESS;
uint64_t group_id = 0;
share::ObGroupName group_name;
group_name.set_value(consumer_group);
ObResourceMappingRuleManager &rule_mgr = G_RES_MGR.get_mapping_rule_mgr();
ObRefHolder<ObTenantIOManager> tenant_holder;
// 删除directive, IO层代表对应的group资源为0,0,0, 但group对应的数据结构不会被释放以防用户后续复用
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_CONFIG;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(rule_mgr.get_group_id_by_name(tenant_id, group_name, group_id))) {
if (OB_HASH_NOT_EXIST == ret) {
//创建directive后立刻删除，可能还没有被刷到存储层或plan未生效，此时不再进行后续操作
ret = OB_SUCCESS;
LOG_INFO("delete directive success with no_releated_io_module", K(consumer_group), K(tenant_id));
} else {
LOG_WARN("fail get group id", K(ret), K(group_id), K(group_name));
}
} else if (OB_UNLIKELY(!is_resource_manager_group(group_id))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid group id", K(ret), K(group_id), K(group_name));
} else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant_holder.get_ptr()->reset_consumer_group_config(group_id))) {
LOG_WARN("reset consumer group iops failed", K(ret), K(group_id), K(tenant_id), K(group_id));
} else {
LOG_INFO("group iops control has reset, delete directive success", K(consumer_group), K(tenant_id), K(group_id));
}
return ret;
}
int ObCgroupCtrl::delete_group_iops(const uint64_t tenant_id,
const common::ObString &consumer_group)
{
int ret = OB_SUCCESS;
uint64_t group_id = 0;
share::ObGroupName group_name;
group_name.set_value(consumer_group);
ObResourceMappingRuleManager &rule_mgr = G_RES_MGR.get_mapping_rule_mgr();
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_CONFIG;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(rule_mgr.get_group_id_by_name(tenant_id, group_name, group_id))) {
if (OB_HASH_NOT_EXIST == ret) {
//创建group后立刻删除，可能还没有被刷到存储层或plan未生效，此时不再进行后续操作
ret = OB_SUCCESS;
LOG_INFO("delete group success with no_releated_io_module", K(consumer_group), K(tenant_id));
} else {
LOG_WARN("fail get group id", K(ret), K(group_id), K(group_name));
}
} else if (OB_UNLIKELY(!is_resource_manager_group(group_id))) {
//OTHER_GROUPS and all cannot be deleted
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid group id", K(ret), K(group_id), K(group_name));
} else {
ObRefHolder<ObTenantIOManager> tenant_holder;
if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant_holder.get_ptr()->delete_consumer_group_config(group_id))) {
LOG_WARN("stop group iops control failed", K(ret), K(tenant_id), K(group_id));
} else {
LOG_INFO("stop group iops_ctrl success when delete group", K(consumer_group), K(tenant_id), K(group_id));
}
}
return ret;
}
int ObCgroupCtrl::init_cgroup_root_dir_(const char *cgroup_path)
{
int ret = OB_SUCCESS;

13
src/share/resource_manager/ob_cgroup_ctrl.h
View File

@ -131,6 +131,7 @@ public:
const int64_t max_net_bandwidth_percent, const int64_t net_bandwidth_weight_percent);
void reset();
bool is_valid() const;
TO_STRING_KV(K_(group_name), K_(min_percent), K_(max_percent), K_(weight_percent), K_(max_net_bandwidth_percent), K_(net_bandwidth_weight_percent));
public:
const char *group_name_;
uint64_t min_percent_;
@ -178,18 +179,6 @@ public:
// 获取某个cgroup组的cpuacct.usage, 即cpu time
int get_cpu_time(const uint64_t tenant_id, int64_t &cpu_time, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = "");
int get_throttled_time(const uint64_t tenant_id, int64_t &throttled_time, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = "");
// 设定指定租户cgroup组的iops，直接更新到租户io_config
int set_group_iops(const uint64_t tenant_id,
const int64_t group_id,
const ObGroupIOInfo &group_io);
// 删除正在使用的plan反应到IO层：重置所有IOPS
int reset_all_group_iops(const uint64_t tenant_id);
// 删除directive反应到IO层：重置IOPS
int reset_group_iops(const uint64_t tenant_id,
const common::ObString &consumer_group);
// 删除group反应到IO层：停用对应的group结构
int delete_group_iops(const uint64_t tenant_id,
const common::ObString &consumer_group);
int get_group_info_by_group_id(const uint64_t tenant_id,
uint64_t group_id,
share::ObGroupName &group_name);

55
src/share/resource_manager/ob_resource_manager_proxy.cpp
View File

@ -20,12 +20,7 @@
#include "lib/mysqlclient/ob_mysql_proxy.h"
#include "share/ob_errno.h"
#include "share/schema/ob_schema_utils.h"
#include "share/resource_manager/ob_resource_plan_manager.h"
#include "share/resource_manager/ob_cgroup_ctrl.h"
#include "share/resource_manager/ob_resource_manager.h"
#include "share/inner_table/ob_inner_table_schema_constants.h"
#include "share/resource_manager/ob_resource_mapping_rule_manager.h"
#include "share/io/ob_io_manager.h"
#include "common/ob_timeout_ctx.h"
#include "observer/ob_sql_client_decorator.h"
#include "observer/ob_server_struct.h"
@ -174,22 +169,6 @@ int ObResourceManagerProxy::delete_plan(
}
}
}
if (OB_SUCC(ret)) {
ObResMgrVarcharValue cur_plan;
ObResourcePlanManager &plan_mgr = G_RES_MGR.get_plan_mgr();
if (OB_FAIL(plan_mgr.get_cur_plan(tenant_id, cur_plan))) {
LOG_WARN("get cur plan failed", K(ret), K(tenant_id), K(cur_plan));
} else if (cur_plan.get_value() != plan) {
//删除非当前使用plan，do nothing
} else {
//删除当前使用的plan，把当前所有IO资源置空
if (OB_FAIL(GCTX.cgroup_ctrl_->reset_all_group_iops(tenant_id))) {
LOG_WARN("reset cur plan group directive failed",K(plan), K(ret));
} else {
LOG_INFO("reset cur plan group directive success",K(plan), K(ret));
}
}
}
return ret;
}
@ -346,31 +325,6 @@ int ObResourceManagerProxy::delete_consumer_group(
}
}
}
if (OB_SUCC(ret)) {
// 在这里inner sql之后就stop io_control的原因是，无法从内部表读到被删除group的信息
uint64_t group_id = 0;
share::ObGroupName group_name;
group_name.set_value(consumer_group);
ObResourceMappingRuleManager &rule_mgr = G_RES_MGR.get_mapping_rule_mgr();
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_CONFIG;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(rule_mgr.get_group_id_by_name(tenant_id, group_name, group_id))) {
if (OB_HASH_NOT_EXIST == ret) {
//创建group后立刻删除，可能还没有被刷到存储层或plan未生效，此时不再进行后续操作
ret = OB_SUCCESS;
LOG_INFO("delete group success with no_releated_io_module", K(consumer_group), K(tenant_id));
} else {
LOG_WARN("fail get group id", K(ret), K(group_id), K(consumer_group));
}
} else if (OB_FAIL(GCTX.cgroup_ctrl_->delete_group_iops(tenant_id, consumer_group))) {
LOG_WARN("fail to stop cur iops isolation", K(ret), K(tenant_id), K(consumer_group));
} else if (OB_FAIL(GCTX.cgroup_ctrl_->remove_both_cgroup(
tenant_id, group_id, GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) {
LOG_WARN("fail to remove group cgroup", K(ret), K(tenant_id), K(consumer_group), K(group_id));
}
}
return ret;
}
@ -1256,15 +1210,6 @@ int ObResourceManagerProxy::delete_plan_directive(
}
}
}
if (OB_SUCC(ret)) {
// 在这里inner sql之后就stop的原因是， 无法从内部表读到被删除group的信息
if (OB_FAIL(GCTX.cgroup_ctrl_->reset_group_iops(
tenant_id,
group))) {
LOG_WARN("reset deleted group directive failed", K(ret), K(group));
}
}
return ret;
}

1
src/share/resource_manager/ob_resource_manager_proxy.h
View File

@ -16,7 +16,6 @@
#include "lib/ob_define.h"
#include "lib/container/ob_iarray.h"
#include "lib/utility/ob_macro_utils.h"
#include "share/resource_manager/ob_resource_plan_info.h"
namespace oceanbase
{

107
src/share/resource_manager/ob_resource_mapping_rule_manager.cpp
View File

@ -15,6 +15,7 @@
#include "lib/string/ob_string.h"
#include "share/resource_manager/ob_resource_manager_proxy.h"
#include "share/resource_manager/ob_cgroup_ctrl.h"
#include "share/io/ob_io_manager.h"
#include "observer/ob_server_struct.h"
@ -59,7 +60,7 @@ int ObResourceMappingRuleManager::refresh_group_mapping_rule(const uint64_t tena
for (int64_t i = 0; i < rules.count() && OB_SUCC(ret); ++i) {
ObResourceUserMappingRule &rule = rules.at(i);
if (OB_FAIL(group_id_name_map_.set_refactored(
combine_two_ids(rule.tenant_id_, rule.group_id_),
ObTenantGroupIdKey(rule.tenant_id_, rule.group_id_),
rule.group_name_,
0 /* don't overwrite */))) {
if (OB_HASH_EXIST == ret) {
@ -78,6 +79,54 @@ int ObResourceMappingRuleManager::refresh_group_mapping_rule(const uint64_t tena
}
}
}
(void)clear_deleted_group(tenant_id, rules);
}
return ret;
}
int ObResourceMappingRuleManager::clear_deleted_group(
const uint64_t tenant_id, const ObResourceUserMappingRuleSet &rules)
{
int ret = OB_SUCCESS;
ObRefHolder<ObTenantIOManager> tenant_holder;
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_TENANT_ID;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else {
ObSEArray<share::ObTenantGroupIdKey, 16> group_id_keys;
ObSEArray<ObGroupName, 16> group_names;
GetTenantGroupIdNameFunctor functor(tenant_id, group_id_keys, group_names);
if (OB_FAIL(group_id_name_map_.foreach_refactored(functor))) {
LOG_WARN("failed to do foreach", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < group_id_keys.count(); ++i) {
bool is_group_id_found = false;
for (int64_t j = 0; !is_group_id_found && j < rules.count(); ++j) {
const ObResourceUserMappingRule &rule = rules.at(j);
if (share::ObTenantGroupIdKey(rule.tenant_id_, rule.group_id_) == group_id_keys.at(i)) {
is_group_id_found = true;
}
}
if (!is_group_id_found) {
uint64_t deleted_group_id = group_id_keys.at(i).group_id_;
ObGroupName deleted_group_name = group_names.at(i);
LOG_INFO("group_id need to be cleared", K(tenant_id), K(deleted_group_id), K(deleted_group_name));
if (OB_FAIL(GCTX.cgroup_ctrl_->remove_both_cgroup(tenant_id, deleted_group_id))) {
LOG_WARN("failed to remove cgroup", K(ret), K(tenant_id), K(deleted_group_id));
} else if (OB_FAIL(tenant_holder.get_ptr()->delete_consumer_group_config(deleted_group_id))) {
LOG_WARN("delete consumer group config failed", K(ret), K(tenant_id), K(deleted_group_id));
} else if (OB_FAIL(group_id_name_map_.erase_refactored(group_id_keys.at(i)))) {
LOG_WARN("fail erase group mapping from group_map", K(deleted_group_id), K(ret));
} else if (OB_FAIL(
group_name_id_map_.erase_refactored(share::ObTenantGroupKey(tenant_id, deleted_group_name)))) {
LOG_WARN("fail erase group name mapping from group id", K(deleted_group_name), K(ret));
}
}
}
}
}
return ret;
}
@ -185,20 +234,24 @@ int ObResourceMappingRuleManager::clear_resource_function_mapping_rule(const uin
const ObResourceMappingRuleSet &rules)
{
int ret = OB_SUCCESS;
for (common::hash::ObHashMap<share::ObTenantFunctionKey, uint64_t>::const_iterator func_iter = function_rule_map_.begin();
OB_SUCC(ret) && func_iter != function_rule_map_.end(); ++func_iter) {
if (func_iter->first.tenant_id_ == tenant_id && func_iter->second > 0) {
bool hit = false;
for (int64_t i = 0; !hit && i < rules.count(); ++i) {
const ObResourceMappingRule &rule = rules.at(i);
if (share::ObTenantFunctionKey(rule.tenant_id_, rule.value_) == func_iter->first) {
hit = true;
ObSEArray<share::ObTenantFunctionKey, 16> func_keys;
ObSEArray<uint64_t, 16> group_ids;
GetTenantFunctionRuleFunctor functor(tenant_id, func_keys, group_ids);
if (OB_FAIL(function_rule_map_.foreach_refactored(functor))) {
LOG_WARN("failed to do foreach", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < func_keys.count(); ++i) {
bool is_group_id_found = false;
for (int64_t j = 0; !is_group_id_found && j < rules.count(); ++j) {
const ObResourceMappingRule &rule = rules.at(j);
if (share::ObTenantFunctionKey(rule.tenant_id_, rule.value_) == func_keys.at(i)) {
is_group_id_found = true;
}
}
if (!hit) {
LOG_INFO("tenant function need to be cleared", "function", func_iter->first, "group_id", func_iter->second);
if (OB_FAIL(function_rule_map_.set_refactored(func_iter->first, 0, 1/*overwrite*/))) {
LOG_WARN("failed to reset user map", K(ret), K(func_iter->first));
if (!is_group_id_found) {
LOG_INFO("tenant function need to be cleared", "function", func_keys.at(i), "group_id", group_ids.at(i));
if (OB_FAIL(function_rule_map_.erase_refactored(func_keys.at(i)))) {
LOG_WARN("failed to reset user map", K(ret), K(func_keys.at(i)));
}
}
}
@ -210,20 +263,24 @@ int ObResourceMappingRuleManager::clear_resource_user_mapping_rule(const uint64_
const ObResourceUserMappingRuleSet &rules)
{
int ret = OB_SUCCESS;
for (common::hash::ObHashMap<sql::ObTenantUserKey, uint64_t>::const_iterator iter = user_rule_map_.begin();
OB_SUCC(ret) && iter != user_rule_map_.end(); ++iter) {
if (iter->first.tenant_id_ == tenant_id && iter->second > 0) {
bool hit = false;
for (int64_t i = 0; !hit && i < rules.count(); ++i) {
const ObResourceUserMappingRule &rule = rules.at(i);
if (sql::ObTenantUserKey(rule.tenant_id_, rule.user_id_) == iter->first) {
hit = true;
ObSEArray<sql::ObTenantUserKey, 16> user_keys;
ObSEArray<uint64_t, 16> group_ids;
GetTenantUserRuleFunctor functor(tenant_id, user_keys, group_ids);
if (OB_FAIL(user_rule_map_.foreach_refactored(functor))) {
LOG_WARN("failed to do foreach", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < user_keys.count(); ++i) {
bool is_group_id_found = false;
for (int64_t j = 0; !is_group_id_found && j < rules.count(); ++j) {
const ObResourceUserMappingRule &rule = rules.at(j);
if (sql::ObTenantUserKey(rule.tenant_id_, rule.user_id_) == user_keys.at(i)) {
is_group_id_found = true;
}
}
if (!hit) {
LOG_INFO("tenant user group need to be cleared", "user", iter->first, "group_id", iter->second);
if (OB_FAIL(user_rule_map_.set_refactored(iter->first, 0, 1/*overwrite*/))) {
LOG_WARN("failed to reset user map", K(ret), K(iter->first));
if (!is_group_id_found) {
LOG_INFO("tenant user group need to be cleared", "user", user_keys.at(i), "group_id", group_ids.at(i));
if (OB_FAIL(user_rule_map_.erase_refactored(user_keys.at(i)))) {
LOG_WARN("failed to reset user map", K(ret), K(user_keys.at(i)));
}
}
}

91
src/share/resource_manager/ob_resource_mapping_rule_manager.h
View File

@ -41,8 +41,95 @@ public:
virtual ~ObResourceMappingRuleManager() = default;
int init();
int refresh_group_mapping_rule(const uint64_t tenant_id, const common::ObString &plan);
int clear_deleted_group(const uint64_t tenant_id, const ObResourceUserMappingRuleSet &rules);
int refresh_resource_mapping_rule(const uint64_t tenant_id, const common::ObString &plan);
int reset_mapping_rules();
common::hash::ObHashMap<ObTenantGroupIdKey, ObGroupName> &get_group_id_name_map() { return group_id_name_map_; }
class GetTenantGroupIdNameFunctor final
{
public:
GetTenantGroupIdNameFunctor(uint64_t tenant_id, common::ObIArray<share::ObTenantGroupIdKey> &group_id_keys,
common::ObIArray<ObGroupName> &group_names)
: tenant_id_(tenant_id), group_id_keys_(group_id_keys), group_names_(group_names)
{}
~GetTenantGroupIdNameFunctor() = default;
int operator()(hash::HashMapPair<share::ObTenantGroupIdKey, ObGroupName> &kv)
{
int ret = OB_SUCCESS;
if (kv.first.tenant_id_ == tenant_id_) {
if (OB_FAIL(group_id_keys_.push_back(kv.first))) {
LOG_WARN("fail to push back function keys", K(ret), K(kv.first));
} else if (OB_FAIL(group_names_.push_back(kv.second))) {
LOG_WARN("fail to push back group names", K(ret), K(kv.second));
}
}
return ret;
}
private:
uint64_t tenant_id_;
common::ObIArray<share::ObTenantGroupIdKey> &group_id_keys_;
common::ObIArray<ObGroupName> &group_names_;
};
class GetTenantFunctionRuleFunctor final
{
public:
GetTenantFunctionRuleFunctor(uint64_t tenant_id, common::ObIArray<share::ObTenantFunctionKey> &func_keys,
common::ObIArray<uint64_t> &group_ids)
: tenant_id_(tenant_id), func_keys_(func_keys), group_ids_(group_ids)
{}
~GetTenantFunctionRuleFunctor() = default;
int operator()(hash::HashMapPair<share::ObTenantFunctionKey, uint64_t> &kv)
{
int ret = OB_SUCCESS;
if (kv.first.tenant_id_ == tenant_id_) {
if (OB_FAIL(func_keys_.push_back(kv.first))) {
LOG_WARN("fail to push back function keys", K(ret), K(kv.first));
} else if (OB_FAIL(group_ids_.push_back(kv.second))) {
LOG_WARN("fail to push back group ids", K(ret), K(kv.second));
}
}
return ret;
}
private:
int64_t tenant_id_;
common::ObIArray<share::ObTenantFunctionKey> &func_keys_;
common::ObIArray<uint64_t> &group_ids_;
};
class GetTenantUserRuleFunctor final
{
public:
GetTenantUserRuleFunctor(
uint64_t tenant_id, common::ObIArray<sql::ObTenantUserKey> &user_keys, common::ObIArray<uint64_t> &group_ids)
: tenant_id_(tenant_id), user_keys_(user_keys), group_ids_(group_ids)
{}
~GetTenantUserRuleFunctor() = default;
int operator()(hash::HashMapPair<sql::ObTenantUserKey, uint64_t> &kv)
{
int ret = OB_SUCCESS;
if (kv.first.tenant_id_ == tenant_id_ && kv.second > 0) {
if (OB_FAIL(user_keys_.push_back(kv.first))) {
LOG_WARN("fail to push back user keys", K(ret), K(kv.first));
} else if (OB_FAIL(group_ids_.push_back(kv.second))) {
LOG_WARN("fail to push back group ids", K(ret), K(kv.second));
}
}
return ret;
}
private:
uint64_t tenant_id_;
common::ObIArray<sql::ObTenantUserKey> &user_keys_;
common::ObIArray<uint64_t> &group_ids_;
};
inline int get_group_id_by_user(const uint64_t tenant_id, uint64_t user_id, uint64_t &group_id)
{
int ret = common::OB_SUCCESS;
@ -103,7 +190,7 @@ public:
const uint64_t group_id,
ObGroupName &group_name)
{
int ret = group_id_name_map_.get_refactored(common::combine_two_ids(tenant_id, group_id), group_name);
int ret = group_id_name_map_.get_refactored(share::ObTenantGroupIdKey(tenant_id, group_id), group_name);
return ret;
}
@ -145,7 +232,7 @@ private:
// 将 function 映射到 group id，用于后台线程快速确定后台 session 所属 cgroup
common::hash::ObHashMap<share::ObTenantFunctionKey, uint64_t> function_rule_map_;
// 将 group_id 映射到 group_name, 用于快速更新 cgroup fs 目录(包括user和function使用的group)
common::hash::ObHashMap<uint64_t, ObGroupName> group_id_name_map_;
common::hash::ObHashMap<ObTenantGroupIdKey, ObGroupName> group_id_name_map_;
// 将 group_name 映射到 group_id, 用于快速根据group_name找到id(主要是用于io控制)
common::hash::ObHashMap<share::ObTenantGroupKey, uint64_t> group_name_id_map_;
DISALLOW_COPY_AND_ASSIGN(ObResourceMappingRuleManager);

50
src/share/resource_manager/ob_resource_plan_info.h
View File

@ -166,6 +166,54 @@ public:
ObResMgrVarcharValue func_name_;
};
// ObTenantGroupIdKey
class ObTenantGroupIdKey {
public:
ObTenantGroupIdKey() : tenant_id_(OB_INVALID_TENANT_ID), group_id_(OB_INVALID_ID)
{}
ObTenantGroupIdKey(const uint64_t tenant_id, const uint64_t group_id) :
tenant_id_(tenant_id), group_id_(group_id)
{}
int assign(const ObTenantGroupIdKey &other)
{
tenant_id_ = other.tenant_id_;
group_id_ = other.group_id_;
return common::OB_SUCCESS;
}
uint64_t hash() const
{
uint64_t hash_val = 0;
hash_val = common::murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val);
hash_val = common::murmurhash(&group_id_, sizeof(group_id_), hash_val);
return hash_val;
}
int hash(uint64_t &hash_val) const
{
hash_val = hash();
return common::OB_SUCCESS;
}
int compare(const ObTenantGroupIdKey& r) const
{
int cmp = 0;
if (tenant_id_ < r.tenant_id_) {
cmp = -1;
} else if (tenant_id_ == r.tenant_id_) {
cmp = group_id_ < r.group_id_ ? -1 : (group_id_ > r.group_id_ ? 1 : 0);
} else {
cmp = 1;
}
return cmp;
}
bool operator== (const ObTenantGroupIdKey &other) const { return 0 == compare(other); }
bool operator!=(const ObTenantGroupIdKey &other) const { return !operator==(other); }
bool operator<(const ObTenantGroupIdKey &other) const { return -1 == compare(other); }
TO_STRING_KV(K_(tenant_id), K_(group_id));
public:
uint64_t tenant_id_;
uint64_t group_id_;
};
class ObPlanDirective
{
public:
@ -266,7 +314,7 @@ public:
private:
DISALLOW_COPY_AND_ASSIGN(ObPlanDirective);
};
typedef common::ObSEArray<ObPlanDirective, 8> ObPlanDirectiveSet;
class ObResourceMappingRule
{
public:

85
src/share/resource_manager/ob_resource_plan_manager.cpp
View File

@ -15,6 +15,7 @@
#include "lib/string/ob_string.h"
#include "share/io/ob_io_manager.h"
#include "share/resource_manager/ob_resource_manager_proxy.h"
#include "share/resource_manager/ob_resource_manager.h"
#include "share/resource_manager/ob_cgroup_ctrl.h"
#include "observer/ob_server_struct.h"
#include "observer/omt/ob_multi_tenant.h"
@ -57,12 +58,6 @@ int ObResourcePlanManager::switch_resource_plan(const uint64_t tenant_id, ObStri
LOG_WARN("get plan failed", K(ret), K(tenant_id));
}
} else if (origin_plan != cur_plan) {
// switch plan，reset 原来plan下对应directive的io资源
ObResourceManagerProxy proxy;
if (OB_FAIL(GCTX.cgroup_ctrl_->reset_all_group_iops(tenant_id))) {
LOG_ERROR("reset old plan group directive failed", K(tenant_id), K(ret));
}
if (OB_SUCC(ret)) {
if (OB_FAIL(tenant_plan_map_.set_refactored(tenant_id, cur_plan, 1))) { // overrite
LOG_WARN("set plan failed", K(ret), K(tenant_id));
@ -152,7 +147,6 @@ int ObResourcePlanManager::refresh_resource_plan(const uint64_t tenant_id, ObStr
ObPlanDirective other_directive; // for OTHER_GROUPS
other_directive.set_group_id(0);
other_directive.set_tenant_id(tenant_id);
// 首先check plan是否发生了切换，如果plan切换那么原plan中资源设置先清零
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_TENANT_ID;
LOG_WARN("invalid config", K(ret), K(tenant_id));
@ -181,6 +175,7 @@ int ObResourcePlanManager::refresh_resource_plan(const uint64_t tenant_id, ObStr
} else if (OB_FAIL(flush_directive_to_cgroup_fs(directives))) { // for CPU
LOG_WARN("fail flush directive to cgroup fs", K(ret));
}
(void) clear_deleted_directives(tenant_id, directives);
}
if (OB_SUCC(ret)) {
if (REACH_TIME_INTERVAL(10 * 1000 * 1000L)) { // 10s
@ -370,9 +365,12 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant
ObPlanDirective &other_group_directive)
{
int ret = OB_SUCCESS;
ObRefHolder<ObTenantIOManager> tenant_holder;
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_TENANT_ID;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < directives.count(); ++i) {
const ObPlanDirective &cur_directive = directives.at(i);
@ -380,14 +378,18 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant
if (OB_FAIL(cur_io_info.init(cur_directive.group_name_.get_value().ptr(), cur_directive.min_iops_, cur_directive.max_iops_, cur_directive.weight_iops_,
cur_directive.max_net_bandwidth_, cur_directive.net_bandwidth_weight_))) {
LOG_ERROR("fail init group io info", K(cur_directive), K(ret));
} else if (OB_FAIL(GCTX.cgroup_ctrl_->set_group_iops(
cur_directive.tenant_id_,
cur_directive.group_id_,
cur_io_info))) {
} else if (OB_FAIL(tenant_holder.get_ptr()->modify_io_config(cur_directive.group_id_,
cur_io_info.group_name_,
cur_io_info.min_percent_,
cur_io_info.max_percent_,
cur_io_info.weight_percent_,
cur_io_info.max_net_bandwidth_percent_,
cur_io_info.net_bandwidth_weight_percent_))) {
LOG_ERROR("fail set iops. tenant isolation function may not functional!!",
K(cur_directive), K(ret));
} else {
LOG_INFO("set group iops", K(ret), K(tenant_id), K(cur_directive.group_id_), K(cur_io_info));
}
// ignore ret, continue
}
if (OB_SUCC(ret)) {
share::ObGroupIOInfo other_io_info;
@ -397,13 +399,16 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant
other_group_directive.weight_iops_,
other_group_directive.max_net_bandwidth_, other_group_directive.net_bandwidth_weight_))) {
LOG_ERROR("fail init other group io info", K(other_group_directive), K(ret));
} else if (OB_FAIL(GCTX.cgroup_ctrl_->set_group_iops(
other_group_directive.tenant_id_,
other_group_directive.group_id_,
other_io_info))) {
} else if (OB_FAIL(tenant_holder.get_ptr()->modify_io_config(other_group_directive.group_id_,
other_io_info.group_name_,
other_io_info.min_percent_,
other_io_info.max_percent_,
other_io_info.weight_percent_,
other_io_info.max_net_bandwidth_percent_,
other_io_info.net_bandwidth_weight_percent_))) {
LOG_ERROR("fail set iops. tenant isolation function may not functional!!",
K(other_group_directive), K(ret));
} else if (OB_FAIL(refresh_tenant_group_io_config(tenant_id))) {
} else if (OB_FAIL(tenant_holder.get_ptr()->refresh_group_io_config())) {
LOG_WARN("refresh tenant io config failed", K(ret), K(tenant_id));
}
}
@ -411,16 +416,56 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant
return ret;
}
int ObResourcePlanManager::refresh_tenant_group_io_config(const uint64_t tenant_id) {
int ObResourcePlanManager::clear_deleted_directives(const uint64_t tenant_id,
ObPlanDirectiveSet &directives)
{
int ret = OB_SUCCESS;
ObRefHolder<ObTenantIOManager> tenant_holder;
if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
ret = OB_INVALID_TENANT_ID;
LOG_WARN("invalid config", K(ret), K(tenant_id));
} else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) {
LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant_holder.get_ptr()->refresh_group_io_config())) {
LOG_WARN("refresh group io config failed", K(ret), K(tenant_id));
}
if (OB_SUCC(ret)) {
common::ObSEArray<share::ObTenantGroupIdKey, 16> group_id_keys;
common::ObSEArray<ObGroupName, 16> group_names;
common::hash::ObHashMap<ObTenantGroupIdKey, ObGroupName> &group_id_name_map =
G_RES_MGR.get_mapping_rule_mgr().get_group_id_name_map();
ObResourceMappingRuleManager::GetTenantGroupIdNameFunctor functor(tenant_id, group_id_keys, group_names);
if (OB_FAIL(group_id_name_map.foreach_refactored(functor))) {
LOG_WARN("failed to do foreach", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < group_id_keys.count(); ++i) {
bool is_group_id_found = false;
for (int64_t j = 0; !is_group_id_found && j < directives.count(); ++j) {
const share::ObPlanDirective &cur_directive = directives.at(j);
if (cur_directive.group_id_ == group_id_keys.at(i).group_id_) {
is_group_id_found = true;
}
}
if (!is_group_id_found) {
const uint64_t deleted_group_id = group_id_keys.at(i).group_id_;
LOG_INFO("directive need to be cleared", K(tenant_id), K(deleted_group_id));
if (OB_FAIL(tenant_holder.get_ptr()->reset_consumer_group_config(deleted_group_id))) {
LOG_WARN("reset consumer group config failed", K(ret), K(deleted_group_id));
} else if (OB_FAIL(GCTX.cgroup_ctrl_->set_both_cpu_shares(tenant_id,
1,
deleted_group_id,
GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) {
LOG_WARN("fail to set cpu share", K(ret), K(tenant_id), K(deleted_group_id));
} else if (OB_FAIL(GCTX.cgroup_ctrl_->set_both_cpu_cfs_quota(tenant_id,
-1,
deleted_group_id,
GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) {
LOG_WARN("fail to set cpu quota", K(ret), K(tenant_id), K(deleted_group_id));
}
}
}
}
}
return ret;
}

4
src/share/resource_manager/ob_resource_plan_manager.h
View File

@ -31,8 +31,6 @@ static constexpr int64_t OTHER_GROUPS_IOPS_WEIGHT = 100L;
static constexpr int64_t OTHER_GROUPS_NET_BANDWIDTH_WEIGHT = 100L;
class ObResourcePlanManager
{
public:
typedef common::ObSEArray<ObPlanDirective, 8> ObPlanDirectiveSet;
public:
ObResourcePlanManager() : tenant_plan_map_(), background_quota_(INT32_MAX)
{}
@ -48,13 +46,13 @@ private:
int flush_directive_to_iops_control(const uint64_t tenant_id,
ObPlanDirectiveSet &directives,
ObPlanDirective &other_group_directive);
int clear_deleted_directives(const uint64_t tenant_id, ObPlanDirectiveSet &directives);
int normalize_iops_directives(const uint64_t tenant_id,
ObPlanDirectiveSet &directives,
ObPlanDirective &other_group_directive);
int normalize_net_bandwidth_directives(const uint64_t tenant_id,
ObPlanDirectiveSet &directives,
ObPlanDirective &other_group_directive);
int refresh_tenant_group_io_config(const uint64_t tenant_id);
common::hash::ObHashMap<uint64_t, ObResMgrVarcharValue> tenant_plan_map_;
int32_t background_quota_;
/* variables */