diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 91e982a84..62495779a 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -1153,11 +1153,6 @@ void ObTenant::destroy() DESTROY_ENTITY(ctx_); ctx_ = nullptr; } - if (cgroup_ctrl_.is_valid() && - OB_TMP_FAIL(cgroup_ctrl_.remove_both_cgroup( - id_, OB_INVALID_GROUP_ID, GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) { - LOG_WARN_RET(tmp_ret, "remove tenant cgroup failed", K(tmp_ret), K_(id)); - } group_map_.destroy_group(); ObTenantSwitchGuard guard(this); print_all_thread("TENANT_BEFORE_DESTROY", id_); @@ -1176,6 +1171,11 @@ void ObTenant::destroy() common::ob_delete(mtl_init_ctx_); mtl_init_ctx_ = nullptr; } + if (cgroup_ctrl_.is_valid() && + OB_TMP_FAIL(cgroup_ctrl_.remove_both_cgroup( + id_, OB_INVALID_GROUP_ID, GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) { + LOG_WARN_RET(tmp_ret, "remove tenant cgroup failed", K(tmp_ret), K_(id)); + } } void ObTenant::set_unit_max_cpu(double cpu) @@ -1648,8 +1648,7 @@ void ObTenant::print_throttled_time() const uint64_t MODE_CNT = static_cast(ObIOMode::MAX_MODE) + 1; for (int64_t i = 0; i < tenant_holder.get_ptr()->get_group_num(); i++) { uint64_t group_config_index = i * MODE_CNT; - if (!tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).deleted_ && - !tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).cleared_) { + if (!tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).deleted_) { uint64_t group_id = tenant_holder.get_ptr()->get_io_config().group_configs_.at(group_config_index).group_id_; if (OB_TMP_FAIL(tenant_holder.get_ptr()->get_throttled_time(group_id, group_throttled_time))) { LOG_WARN_RET(tmp_ret, "get throttled time failed", K(tmp_ret), K(group_id)); diff --git a/src/observer/virtual_table/ob_all_virtual_io_status.cpp b/src/observer/virtual_table/ob_all_virtual_io_status.cpp index a1f3c849a..59fcc359c 100644 --- a/src/observer/virtual_table/ob_all_virtual_io_status.cpp +++ b/src/observer/virtual_table/ob_all_virtual_io_status.cpp @@ -395,8 +395,7 @@ int ObAllVirtualIOQuota::record_user_group(const uint64_t tenant_id, ObIOUsage & for (int64_t i = 0; i < info.count(); ++i) { if (OB_TMP_FAIL(oceanbase::common::transform_usage_index_to_group_config_index(i, group_config_index))) { } else if (group_config_index >= io_config.group_configs_.count()) { - } else if (io_config.group_configs_.at(group_config_index).deleted_ || - io_config.group_configs_.at(group_config_index).cleared_) { + } else if (io_config.group_configs_.at(group_config_index).deleted_) { } else if (info.at(i).avg_byte_ > std::numeric_limits::epsilon()) { QuotaInfo item; ObIOMode mode = static_cast(group_config_index % MODE_COUNT); @@ -676,7 +675,7 @@ int ObAllVirtualIOScheduler::init(const common::ObAddr &addr) const ObTenantIOConfig &io_config = tenant_holder.get_ptr()->get_io_config(); int64_t group_num = io_config.group_configs_.count(); for (int64_t index = 0; OB_SUCC(ret) && index < group_num; ++index) { - if (io_config.group_configs_.at(index).deleted_ || io_config.group_configs_.at(index).cleared_) { + if (io_config.group_configs_.at(index).deleted_) { continue; } ScheduleInfo item; diff --git a/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp b/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp index c205ce27e..09f1a999a 100644 --- a/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp +++ b/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp @@ -92,7 +92,7 @@ int ObPlDBMSResourceManager::delete_plan( } if (OB_SUCC(ret)) { if (OB_FAIL(proxy.delete_plan(tenant_id, plan))) { - LOG_WARN("fail create plan", K(tenant_id), K(plan), K(ret)); + LOG_WARN("fail delete plan", K(tenant_id), K(plan), K(ret)); } } return ret; diff --git a/src/share/resource_manager/ob_cgroup_ctrl.cpp b/src/share/resource_manager/ob_cgroup_ctrl.cpp index dbc01f149..3d68ecb3a 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.cpp +++ b/src/share/resource_manager/ob_cgroup_ctrl.cpp @@ -801,122 +801,6 @@ int ObCgroupCtrl::get_throttled_time(const uint64_t tenant_id, int64_t &throttle return ret; } -int ObCgroupCtrl::set_group_iops(const uint64_t tenant_id, - const int64_t group_id, - const ObGroupIOInfo &group_io) -{ - int ret = OB_SUCCESS; - - ObRefHolder tenant_holder; - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { - ret = OB_INVALID_CONFIG; - LOG_WARN("invalid config", K(ret), K(tenant_id)); - } else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { - LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_holder.get_ptr()->modify_io_config(group_id, - group_io.group_name_, - group_io.min_percent_, - group_io.max_percent_, - group_io.weight_percent_, - group_io.max_net_bandwidth_percent_, - group_io.net_bandwidth_weight_percent_))) { - LOG_WARN("modify consumer group iops failed", K(ret), K(group_id), K(tenant_id), K(group_id)); - } - return ret; -} - -int ObCgroupCtrl::reset_all_group_iops(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - - ObRefHolder tenant_holder; - // 删除plan, IO层代表对应的所有group资源为0,0,0, 但group对应的数据结构不会被释放以防用户后续复用 - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { - ret = OB_INVALID_CONFIG; - LOG_WARN("invalid config", K(ret), K(tenant_id)); - } else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { - LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_holder.get_ptr()->reset_all_group_config())) { - LOG_WARN("reset consumer group iops failed", K(ret), K(tenant_id)); - } else { - LOG_INFO("group iops control has reset, delete cur plan success", K(tenant_id)); - } - return ret; -} - -int ObCgroupCtrl::reset_group_iops(const uint64_t tenant_id, - const common::ObString &consumer_group) -{ - int ret = OB_SUCCESS; - - uint64_t group_id = 0; - share::ObGroupName group_name; - group_name.set_value(consumer_group); - ObResourceMappingRuleManager &rule_mgr = G_RES_MGR.get_mapping_rule_mgr(); - ObRefHolder tenant_holder; - - // 删除directive, IO层代表对应的group资源为0,0,0, 但group对应的数据结构不会被释放以防用户后续复用 - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { - ret = OB_INVALID_CONFIG; - LOG_WARN("invalid config", K(ret), K(tenant_id)); - } else if (OB_FAIL(rule_mgr.get_group_id_by_name(tenant_id, group_name, group_id))) { - if (OB_HASH_NOT_EXIST == ret) { - //创建directive后立刻删除，可能还没有被刷到存储层或plan未生效，此时不再进行后续操作 - ret = OB_SUCCESS; - LOG_INFO("delete directive success with no_releated_io_module", K(consumer_group), K(tenant_id)); - } else { - LOG_WARN("fail get group id", K(ret), K(group_id), K(group_name)); - } - } else if (OB_UNLIKELY(!is_resource_manager_group(group_id))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid group id", K(ret), K(group_id), K(group_name)); - } else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { - LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_holder.get_ptr()->reset_consumer_group_config(group_id))) { - LOG_WARN("reset consumer group iops failed", K(ret), K(group_id), K(tenant_id), K(group_id)); - } else { - LOG_INFO("group iops control has reset, delete directive success", K(consumer_group), K(tenant_id), K(group_id)); - } - return ret; -} - -int ObCgroupCtrl::delete_group_iops(const uint64_t tenant_id, - const common::ObString &consumer_group) -{ - int ret = OB_SUCCESS; - - uint64_t group_id = 0; - share::ObGroupName group_name; - group_name.set_value(consumer_group); - ObResourceMappingRuleManager &rule_mgr = G_RES_MGR.get_mapping_rule_mgr(); - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { - ret = OB_INVALID_CONFIG; - LOG_WARN("invalid config", K(ret), K(tenant_id)); - } else if (OB_FAIL(rule_mgr.get_group_id_by_name(tenant_id, group_name, group_id))) { - if (OB_HASH_NOT_EXIST == ret) { - //创建group后立刻删除，可能还没有被刷到存储层或plan未生效，此时不再进行后续操作 - ret = OB_SUCCESS; - LOG_INFO("delete group success with no_releated_io_module", K(consumer_group), K(tenant_id)); - } else { - LOG_WARN("fail get group id", K(ret), K(group_id), K(group_name)); - } - } else if (OB_UNLIKELY(!is_resource_manager_group(group_id))) { - //OTHER_GROUPS and all cannot be deleted - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid group id", K(ret), K(group_id), K(group_name)); - } else { - ObRefHolder tenant_holder; - if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { - LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_holder.get_ptr()->delete_consumer_group_config(group_id))) { - LOG_WARN("stop group iops control failed", K(ret), K(tenant_id), K(group_id)); - } else { - LOG_INFO("stop group iops_ctrl success when delete group", K(consumer_group), K(tenant_id), K(group_id)); - } - } - return ret; -} - int ObCgroupCtrl::init_cgroup_root_dir_(const char *cgroup_path) { int ret = OB_SUCCESS; diff --git a/src/share/resource_manager/ob_cgroup_ctrl.h b/src/share/resource_manager/ob_cgroup_ctrl.h index 4145f9f87..64641dfa7 100644 --- a/src/share/resource_manager/ob_cgroup_ctrl.h +++ b/src/share/resource_manager/ob_cgroup_ctrl.h @@ -131,6 +131,7 @@ public: const int64_t max_net_bandwidth_percent, const int64_t net_bandwidth_weight_percent); void reset(); bool is_valid() const; + TO_STRING_KV(K_(group_name), K_(min_percent), K_(max_percent), K_(weight_percent), K_(max_net_bandwidth_percent), K_(net_bandwidth_weight_percent)); public: const char *group_name_; uint64_t min_percent_; @@ -178,18 +179,6 @@ public: // 获取某个cgroup组的cpuacct.usage, 即cpu time int get_cpu_time(const uint64_t tenant_id, int64_t &cpu_time, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = ""); int get_throttled_time(const uint64_t tenant_id, int64_t &throttled_time, const uint64_t group_id = OB_INVALID_GROUP_ID, const char *base_path = ""); - // 设定指定租户cgroup组的iops，直接更新到租户io_config - int set_group_iops(const uint64_t tenant_id, - const int64_t group_id, - const ObGroupIOInfo &group_io); - // 删除正在使用的plan反应到IO层：重置所有IOPS - int reset_all_group_iops(const uint64_t tenant_id); - // 删除directive反应到IO层：重置IOPS - int reset_group_iops(const uint64_t tenant_id, - const common::ObString &consumer_group); - // 删除group反应到IO层：停用对应的group结构 - int delete_group_iops(const uint64_t tenant_id, - const common::ObString &consumer_group); int get_group_info_by_group_id(const uint64_t tenant_id, uint64_t group_id, share::ObGroupName &group_name); diff --git a/src/share/resource_manager/ob_resource_manager_proxy.cpp b/src/share/resource_manager/ob_resource_manager_proxy.cpp index 884b2b17a..9a549aa6e 100644 --- a/src/share/resource_manager/ob_resource_manager_proxy.cpp +++ b/src/share/resource_manager/ob_resource_manager_proxy.cpp @@ -20,12 +20,7 @@ #include "lib/mysqlclient/ob_mysql_proxy.h" #include "share/ob_errno.h" #include "share/schema/ob_schema_utils.h" -#include "share/resource_manager/ob_resource_plan_manager.h" -#include "share/resource_manager/ob_cgroup_ctrl.h" -#include "share/resource_manager/ob_resource_manager.h" #include "share/inner_table/ob_inner_table_schema_constants.h" -#include "share/resource_manager/ob_resource_mapping_rule_manager.h" -#include "share/io/ob_io_manager.h" #include "common/ob_timeout_ctx.h" #include "observer/ob_sql_client_decorator.h" #include "observer/ob_server_struct.h" @@ -174,22 +169,6 @@ int ObResourceManagerProxy::delete_plan( } } } - if (OB_SUCC(ret)) { - ObResMgrVarcharValue cur_plan; - ObResourcePlanManager &plan_mgr = G_RES_MGR.get_plan_mgr(); - if (OB_FAIL(plan_mgr.get_cur_plan(tenant_id, cur_plan))) { - LOG_WARN("get cur plan failed", K(ret), K(tenant_id), K(cur_plan)); - } else if (cur_plan.get_value() != plan) { - //删除非当前使用plan，do nothing - } else { - //删除当前使用的plan，把当前所有IO资源置空 - if (OB_FAIL(GCTX.cgroup_ctrl_->reset_all_group_iops(tenant_id))) { - LOG_WARN("reset cur plan group directive failed",K(plan), K(ret)); - } else { - LOG_INFO("reset cur plan group directive success",K(plan), K(ret)); - } - } - } return ret; } @@ -346,31 +325,6 @@ int ObResourceManagerProxy::delete_consumer_group( } } } - - if (OB_SUCC(ret)) { - // 在这里inner sql之后就stop io_control的原因是，无法从内部表读到被删除group的信息 - uint64_t group_id = 0; - share::ObGroupName group_name; - group_name.set_value(consumer_group); - ObResourceMappingRuleManager &rule_mgr = G_RES_MGR.get_mapping_rule_mgr(); - if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { - ret = OB_INVALID_CONFIG; - LOG_WARN("invalid config", K(ret), K(tenant_id)); - } else if (OB_FAIL(rule_mgr.get_group_id_by_name(tenant_id, group_name, group_id))) { - if (OB_HASH_NOT_EXIST == ret) { - //创建group后立刻删除，可能还没有被刷到存储层或plan未生效，此时不再进行后续操作 - ret = OB_SUCCESS; - LOG_INFO("delete group success with no_releated_io_module", K(consumer_group), K(tenant_id)); - } else { - LOG_WARN("fail get group id", K(ret), K(group_id), K(consumer_group)); - } - } else if (OB_FAIL(GCTX.cgroup_ctrl_->delete_group_iops(tenant_id, consumer_group))) { - LOG_WARN("fail to stop cur iops isolation", K(ret), K(tenant_id), K(consumer_group)); - } else if (OB_FAIL(GCTX.cgroup_ctrl_->remove_both_cgroup( - tenant_id, group_id, GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) { - LOG_WARN("fail to remove group cgroup", K(ret), K(tenant_id), K(consumer_group), K(group_id)); - } - } return ret; } @@ -1256,15 +1210,6 @@ int ObResourceManagerProxy::delete_plan_directive( } } } - - if (OB_SUCC(ret)) { - // 在这里inner sql之后就stop的原因是， 无法从内部表读到被删除group的信息 - if (OB_FAIL(GCTX.cgroup_ctrl_->reset_group_iops( - tenant_id, - group))) { - LOG_WARN("reset deleted group directive failed", K(ret), K(group)); - } - } return ret; } diff --git a/src/share/resource_manager/ob_resource_manager_proxy.h b/src/share/resource_manager/ob_resource_manager_proxy.h index a87e9f38e..5aaf78033 100644 --- a/src/share/resource_manager/ob_resource_manager_proxy.h +++ b/src/share/resource_manager/ob_resource_manager_proxy.h @@ -16,7 +16,6 @@ #include "lib/ob_define.h" #include "lib/container/ob_iarray.h" #include "lib/utility/ob_macro_utils.h" -#include "share/resource_manager/ob_resource_plan_info.h" namespace oceanbase { diff --git a/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp b/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp index c7ba01629..d233d53e0 100644 --- a/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp +++ b/src/share/resource_manager/ob_resource_mapping_rule_manager.cpp @@ -15,6 +15,7 @@ #include "lib/string/ob_string.h" #include "share/resource_manager/ob_resource_manager_proxy.h" #include "share/resource_manager/ob_cgroup_ctrl.h" +#include "share/io/ob_io_manager.h" #include "observer/ob_server_struct.h" @@ -59,7 +60,7 @@ int ObResourceMappingRuleManager::refresh_group_mapping_rule(const uint64_t tena for (int64_t i = 0; i < rules.count() && OB_SUCC(ret); ++i) { ObResourceUserMappingRule &rule = rules.at(i); if (OB_FAIL(group_id_name_map_.set_refactored( - combine_two_ids(rule.tenant_id_, rule.group_id_), + ObTenantGroupIdKey(rule.tenant_id_, rule.group_id_), rule.group_name_, 0 /* don't overwrite */))) { if (OB_HASH_EXIST == ret) { @@ -78,6 +79,54 @@ int ObResourceMappingRuleManager::refresh_group_mapping_rule(const uint64_t tena } } } + (void)clear_deleted_group(tenant_id, rules); + } + return ret; +} + +int ObResourceMappingRuleManager::clear_deleted_group( + const uint64_t tenant_id, const ObResourceUserMappingRuleSet &rules) +{ + int ret = OB_SUCCESS; + + ObRefHolder tenant_holder; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_TENANT_ID; + LOG_WARN("invalid config", K(ret), K(tenant_id)); + } else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { + LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); + } else { + ObSEArray group_id_keys; + ObSEArray group_names; + GetTenantGroupIdNameFunctor functor(tenant_id, group_id_keys, group_names); + if (OB_FAIL(group_id_name_map_.foreach_refactored(functor))) { + LOG_WARN("failed to do foreach", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < group_id_keys.count(); ++i) { + bool is_group_id_found = false; + for (int64_t j = 0; !is_group_id_found && j < rules.count(); ++j) { + const ObResourceUserMappingRule &rule = rules.at(j); + if (share::ObTenantGroupIdKey(rule.tenant_id_, rule.group_id_) == group_id_keys.at(i)) { + is_group_id_found = true; + } + } + if (!is_group_id_found) { + uint64_t deleted_group_id = group_id_keys.at(i).group_id_; + ObGroupName deleted_group_name = group_names.at(i); + LOG_INFO("group_id need to be cleared", K(tenant_id), K(deleted_group_id), K(deleted_group_name)); + if (OB_FAIL(GCTX.cgroup_ctrl_->remove_both_cgroup(tenant_id, deleted_group_id))) { + LOG_WARN("failed to remove cgroup", K(ret), K(tenant_id), K(deleted_group_id)); + } else if (OB_FAIL(tenant_holder.get_ptr()->delete_consumer_group_config(deleted_group_id))) { + LOG_WARN("delete consumer group config failed", K(ret), K(tenant_id), K(deleted_group_id)); + } else if (OB_FAIL(group_id_name_map_.erase_refactored(group_id_keys.at(i)))) { + LOG_WARN("fail erase group mapping from group_map", K(deleted_group_id), K(ret)); + } else if (OB_FAIL( + group_name_id_map_.erase_refactored(share::ObTenantGroupKey(tenant_id, deleted_group_name)))) { + LOG_WARN("fail erase group name mapping from group id", K(deleted_group_name), K(ret)); + } + } + } + } } return ret; } @@ -185,20 +234,24 @@ int ObResourceMappingRuleManager::clear_resource_function_mapping_rule(const uin const ObResourceMappingRuleSet &rules) { int ret = OB_SUCCESS; - for (common::hash::ObHashMap::const_iterator func_iter = function_rule_map_.begin(); - OB_SUCC(ret) && func_iter != function_rule_map_.end(); ++func_iter) { - if (func_iter->first.tenant_id_ == tenant_id && func_iter->second > 0) { - bool hit = false; - for (int64_t i = 0; !hit && i < rules.count(); ++i) { - const ObResourceMappingRule &rule = rules.at(i); - if (share::ObTenantFunctionKey(rule.tenant_id_, rule.value_) == func_iter->first) { - hit = true; + ObSEArray func_keys; + ObSEArray group_ids; + GetTenantFunctionRuleFunctor functor(tenant_id, func_keys, group_ids); + if (OB_FAIL(function_rule_map_.foreach_refactored(functor))) { + LOG_WARN("failed to do foreach", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < func_keys.count(); ++i) { + bool is_group_id_found = false; + for (int64_t j = 0; !is_group_id_found && j < rules.count(); ++j) { + const ObResourceMappingRule &rule = rules.at(j); + if (share::ObTenantFunctionKey(rule.tenant_id_, rule.value_) == func_keys.at(i)) { + is_group_id_found = true; } } - if (!hit) { - LOG_INFO("tenant function need to be cleared", "function", func_iter->first, "group_id", func_iter->second); - if (OB_FAIL(function_rule_map_.set_refactored(func_iter->first, 0, 1/*overwrite*/))) { - LOG_WARN("failed to reset user map", K(ret), K(func_iter->first)); + if (!is_group_id_found) { + LOG_INFO("tenant function need to be cleared", "function", func_keys.at(i), "group_id", group_ids.at(i)); + if (OB_FAIL(function_rule_map_.erase_refactored(func_keys.at(i)))) { + LOG_WARN("failed to reset user map", K(ret), K(func_keys.at(i))); } } } @@ -210,20 +263,24 @@ int ObResourceMappingRuleManager::clear_resource_user_mapping_rule(const uint64_ const ObResourceUserMappingRuleSet &rules) { int ret = OB_SUCCESS; - for (common::hash::ObHashMap::const_iterator iter = user_rule_map_.begin(); - OB_SUCC(ret) && iter != user_rule_map_.end(); ++iter) { - if (iter->first.tenant_id_ == tenant_id && iter->second > 0) { - bool hit = false; - for (int64_t i = 0; !hit && i < rules.count(); ++i) { - const ObResourceUserMappingRule &rule = rules.at(i); - if (sql::ObTenantUserKey(rule.tenant_id_, rule.user_id_) == iter->first) { - hit = true; + ObSEArray user_keys; + ObSEArray group_ids; + GetTenantUserRuleFunctor functor(tenant_id, user_keys, group_ids); + if (OB_FAIL(user_rule_map_.foreach_refactored(functor))) { + LOG_WARN("failed to do foreach", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < user_keys.count(); ++i) { + bool is_group_id_found = false; + for (int64_t j = 0; !is_group_id_found && j < rules.count(); ++j) { + const ObResourceUserMappingRule &rule = rules.at(j); + if (sql::ObTenantUserKey(rule.tenant_id_, rule.user_id_) == user_keys.at(i)) { + is_group_id_found = true; } } - if (!hit) { - LOG_INFO("tenant user group need to be cleared", "user", iter->first, "group_id", iter->second); - if (OB_FAIL(user_rule_map_.set_refactored(iter->first, 0, 1/*overwrite*/))) { - LOG_WARN("failed to reset user map", K(ret), K(iter->first)); + if (!is_group_id_found) { + LOG_INFO("tenant user group need to be cleared", "user", user_keys.at(i), "group_id", group_ids.at(i)); + if (OB_FAIL(user_rule_map_.erase_refactored(user_keys.at(i)))) { + LOG_WARN("failed to reset user map", K(ret), K(user_keys.at(i))); } } } diff --git a/src/share/resource_manager/ob_resource_mapping_rule_manager.h b/src/share/resource_manager/ob_resource_mapping_rule_manager.h index 3c748f205..aaa4a2fcb 100644 --- a/src/share/resource_manager/ob_resource_mapping_rule_manager.h +++ b/src/share/resource_manager/ob_resource_mapping_rule_manager.h @@ -41,8 +41,95 @@ public: virtual ~ObResourceMappingRuleManager() = default; int init(); int refresh_group_mapping_rule(const uint64_t tenant_id, const common::ObString &plan); + int clear_deleted_group(const uint64_t tenant_id, const ObResourceUserMappingRuleSet &rules); int refresh_resource_mapping_rule(const uint64_t tenant_id, const common::ObString &plan); int reset_mapping_rules(); + common::hash::ObHashMap &get_group_id_name_map() { return group_id_name_map_; } + + class GetTenantGroupIdNameFunctor final + { + public: + GetTenantGroupIdNameFunctor(uint64_t tenant_id, common::ObIArray &group_id_keys, + common::ObIArray &group_names) + : tenant_id_(tenant_id), group_id_keys_(group_id_keys), group_names_(group_names) + {} + ~GetTenantGroupIdNameFunctor() = default; + + int operator()(hash::HashMapPair &kv) + { + int ret = OB_SUCCESS; + if (kv.first.tenant_id_ == tenant_id_) { + if (OB_FAIL(group_id_keys_.push_back(kv.first))) { + LOG_WARN("fail to push back function keys", K(ret), K(kv.first)); + } else if (OB_FAIL(group_names_.push_back(kv.second))) { + LOG_WARN("fail to push back group names", K(ret), K(kv.second)); + } + } + return ret; + } + + private: + uint64_t tenant_id_; + common::ObIArray &group_id_keys_; + common::ObIArray &group_names_; + }; + + class GetTenantFunctionRuleFunctor final + { + public: + GetTenantFunctionRuleFunctor(uint64_t tenant_id, common::ObIArray &func_keys, + common::ObIArray &group_ids) + : tenant_id_(tenant_id), func_keys_(func_keys), group_ids_(group_ids) + {} + ~GetTenantFunctionRuleFunctor() = default; + + int operator()(hash::HashMapPair &kv) + { + int ret = OB_SUCCESS; + if (kv.first.tenant_id_ == tenant_id_) { + if (OB_FAIL(func_keys_.push_back(kv.first))) { + LOG_WARN("fail to push back function keys", K(ret), K(kv.first)); + } else if (OB_FAIL(group_ids_.push_back(kv.second))) { + LOG_WARN("fail to push back group ids", K(ret), K(kv.second)); + } + } + return ret; + } + + private: + int64_t tenant_id_; + common::ObIArray &func_keys_; + common::ObIArray &group_ids_; + }; + + class GetTenantUserRuleFunctor final + { + public: + GetTenantUserRuleFunctor( + uint64_t tenant_id, common::ObIArray &user_keys, common::ObIArray &group_ids) + : tenant_id_(tenant_id), user_keys_(user_keys), group_ids_(group_ids) + {} + ~GetTenantUserRuleFunctor() = default; + + int operator()(hash::HashMapPair &kv) + { + int ret = OB_SUCCESS; + if (kv.first.tenant_id_ == tenant_id_ && kv.second > 0) { + if (OB_FAIL(user_keys_.push_back(kv.first))) { + LOG_WARN("fail to push back user keys", K(ret), K(kv.first)); + } else if (OB_FAIL(group_ids_.push_back(kv.second))) { + LOG_WARN("fail to push back group ids", K(ret), K(kv.second)); + } + } + return ret; + } + + private: + uint64_t tenant_id_; + common::ObIArray &user_keys_; + common::ObIArray &group_ids_; + }; + inline int get_group_id_by_user(const uint64_t tenant_id, uint64_t user_id, uint64_t &group_id) { int ret = common::OB_SUCCESS; @@ -103,7 +190,7 @@ public: const uint64_t group_id, ObGroupName &group_name) { - int ret = group_id_name_map_.get_refactored(common::combine_two_ids(tenant_id, group_id), group_name); + int ret = group_id_name_map_.get_refactored(share::ObTenantGroupIdKey(tenant_id, group_id), group_name); return ret; } @@ -145,7 +232,7 @@ private: // 将 function 映射到 group id，用于后台线程快速确定后台 session 所属 cgroup common::hash::ObHashMap function_rule_map_; // 将 group_id 映射到 group_name, 用于快速更新 cgroup fs 目录(包括user和function使用的group) - common::hash::ObHashMap group_id_name_map_; + common::hash::ObHashMap group_id_name_map_; // 将 group_name 映射到 group_id, 用于快速根据group_name找到id(主要是用于io控制) common::hash::ObHashMap group_name_id_map_; DISALLOW_COPY_AND_ASSIGN(ObResourceMappingRuleManager); diff --git a/src/share/resource_manager/ob_resource_plan_info.h b/src/share/resource_manager/ob_resource_plan_info.h index 67aeabb40..0b97315ae 100644 --- a/src/share/resource_manager/ob_resource_plan_info.h +++ b/src/share/resource_manager/ob_resource_plan_info.h @@ -166,6 +166,54 @@ public: ObResMgrVarcharValue func_name_; }; +// ObTenantGroupIdKey +class ObTenantGroupIdKey { +public: + ObTenantGroupIdKey() : tenant_id_(OB_INVALID_TENANT_ID), group_id_(OB_INVALID_ID) + {} + ObTenantGroupIdKey(const uint64_t tenant_id, const uint64_t group_id) : + tenant_id_(tenant_id), group_id_(group_id) + {} + int assign(const ObTenantGroupIdKey &other) + { + tenant_id_ = other.tenant_id_; + group_id_ = other.group_id_; + return common::OB_SUCCESS; + } + uint64_t hash() const + { + uint64_t hash_val = 0; + hash_val = common::murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); + hash_val = common::murmurhash(&group_id_, sizeof(group_id_), hash_val); + return hash_val; + } + int hash(uint64_t &hash_val) const + { + hash_val = hash(); + return common::OB_SUCCESS; + } + int compare(const ObTenantGroupIdKey& r) const + { + int cmp = 0; + if (tenant_id_ < r.tenant_id_) { + cmp = -1; + } else if (tenant_id_ == r.tenant_id_) { + cmp = group_id_ < r.group_id_ ? -1 : (group_id_ > r.group_id_ ? 1 : 0); + } else { + cmp = 1; + } + return cmp; + } + bool operator== (const ObTenantGroupIdKey &other) const { return 0 == compare(other); } + bool operator!=(const ObTenantGroupIdKey &other) const { return !operator==(other); } + bool operator<(const ObTenantGroupIdKey &other) const { return -1 == compare(other); } + TO_STRING_KV(K_(tenant_id), K_(group_id)); + +public: + uint64_t tenant_id_; + uint64_t group_id_; +}; + class ObPlanDirective { public: @@ -266,7 +314,7 @@ public: private: DISALLOW_COPY_AND_ASSIGN(ObPlanDirective); }; - +typedef common::ObSEArray ObPlanDirectiveSet; class ObResourceMappingRule { public: diff --git a/src/share/resource_manager/ob_resource_plan_manager.cpp b/src/share/resource_manager/ob_resource_plan_manager.cpp index 8177cc724..4d5654d9a 100644 --- a/src/share/resource_manager/ob_resource_plan_manager.cpp +++ b/src/share/resource_manager/ob_resource_plan_manager.cpp @@ -15,6 +15,7 @@ #include "lib/string/ob_string.h" #include "share/io/ob_io_manager.h" #include "share/resource_manager/ob_resource_manager_proxy.h" +#include "share/resource_manager/ob_resource_manager.h" #include "share/resource_manager/ob_cgroup_ctrl.h" #include "observer/ob_server_struct.h" #include "observer/omt/ob_multi_tenant.h" @@ -57,12 +58,6 @@ int ObResourcePlanManager::switch_resource_plan(const uint64_t tenant_id, ObStri LOG_WARN("get plan failed", K(ret), K(tenant_id)); } } else if (origin_plan != cur_plan) { - // switch plan，reset 原来plan下对应directive的io资源 - ObResourceManagerProxy proxy; - if (OB_FAIL(GCTX.cgroup_ctrl_->reset_all_group_iops(tenant_id))) { - LOG_ERROR("reset old plan group directive failed", K(tenant_id), K(ret)); - } - if (OB_SUCC(ret)) { if (OB_FAIL(tenant_plan_map_.set_refactored(tenant_id, cur_plan, 1))) { // overrite LOG_WARN("set plan failed", K(ret), K(tenant_id)); @@ -152,7 +147,6 @@ int ObResourcePlanManager::refresh_resource_plan(const uint64_t tenant_id, ObStr ObPlanDirective other_directive; // for OTHER_GROUPS other_directive.set_group_id(0); other_directive.set_tenant_id(tenant_id); - // 首先check plan是否发生了切换，如果plan切换那么原plan中资源设置先清零 if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_TENANT_ID; LOG_WARN("invalid config", K(ret), K(tenant_id)); @@ -181,6 +175,7 @@ int ObResourcePlanManager::refresh_resource_plan(const uint64_t tenant_id, ObStr } else if (OB_FAIL(flush_directive_to_cgroup_fs(directives))) { // for CPU LOG_WARN("fail flush directive to cgroup fs", K(ret)); } + (void) clear_deleted_directives(tenant_id, directives); } if (OB_SUCC(ret)) { if (REACH_TIME_INTERVAL(10 * 1000 * 1000L)) { // 10s @@ -370,9 +365,12 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant ObPlanDirective &other_group_directive) { int ret = OB_SUCCESS; + ObRefHolder tenant_holder; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_TENANT_ID; LOG_WARN("invalid config", K(ret), K(tenant_id)); + } else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { + LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < directives.count(); ++i) { const ObPlanDirective &cur_directive = directives.at(i); @@ -380,14 +378,18 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant if (OB_FAIL(cur_io_info.init(cur_directive.group_name_.get_value().ptr(), cur_directive.min_iops_, cur_directive.max_iops_, cur_directive.weight_iops_, cur_directive.max_net_bandwidth_, cur_directive.net_bandwidth_weight_))) { LOG_ERROR("fail init group io info", K(cur_directive), K(ret)); - } else if (OB_FAIL(GCTX.cgroup_ctrl_->set_group_iops( - cur_directive.tenant_id_, - cur_directive.group_id_, - cur_io_info))) { + } else if (OB_FAIL(tenant_holder.get_ptr()->modify_io_config(cur_directive.group_id_, + cur_io_info.group_name_, + cur_io_info.min_percent_, + cur_io_info.max_percent_, + cur_io_info.weight_percent_, + cur_io_info.max_net_bandwidth_percent_, + cur_io_info.net_bandwidth_weight_percent_))) { LOG_ERROR("fail set iops. tenant isolation function may not functional!!", K(cur_directive), K(ret)); + } else { + LOG_INFO("set group iops", K(ret), K(tenant_id), K(cur_directive.group_id_), K(cur_io_info)); } - // ignore ret, continue } if (OB_SUCC(ret)) { share::ObGroupIOInfo other_io_info; @@ -397,13 +399,16 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant other_group_directive.weight_iops_, other_group_directive.max_net_bandwidth_, other_group_directive.net_bandwidth_weight_))) { LOG_ERROR("fail init other group io info", K(other_group_directive), K(ret)); - } else if (OB_FAIL(GCTX.cgroup_ctrl_->set_group_iops( - other_group_directive.tenant_id_, - other_group_directive.group_id_, - other_io_info))) { + } else if (OB_FAIL(tenant_holder.get_ptr()->modify_io_config(other_group_directive.group_id_, + other_io_info.group_name_, + other_io_info.min_percent_, + other_io_info.max_percent_, + other_io_info.weight_percent_, + other_io_info.max_net_bandwidth_percent_, + other_io_info.net_bandwidth_weight_percent_))) { LOG_ERROR("fail set iops. tenant isolation function may not functional!!", K(other_group_directive), K(ret)); - } else if (OB_FAIL(refresh_tenant_group_io_config(tenant_id))) { + } else if (OB_FAIL(tenant_holder.get_ptr()->refresh_group_io_config())) { LOG_WARN("refresh tenant io config failed", K(ret), K(tenant_id)); } } @@ -411,16 +416,56 @@ int ObResourcePlanManager::flush_directive_to_iops_control(const uint64_t tenant return ret; } -int ObResourcePlanManager::refresh_tenant_group_io_config(const uint64_t tenant_id) { + +int ObResourcePlanManager::clear_deleted_directives(const uint64_t tenant_id, + ObPlanDirectiveSet &directives) +{ int ret = OB_SUCCESS; + ObRefHolder tenant_holder; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { ret = OB_INVALID_TENANT_ID; LOG_WARN("invalid config", K(ret), K(tenant_id)); } else if (OB_FAIL(OB_IO_MANAGER.get_tenant_io_manager(tenant_id, tenant_holder))) { LOG_WARN("get tenant io manager failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_holder.get_ptr()->refresh_group_io_config())) { - LOG_WARN("refresh group io config failed", K(ret), K(tenant_id)); + } + + if (OB_SUCC(ret)) { + common::ObSEArray group_id_keys; + common::ObSEArray group_names; + common::hash::ObHashMap &group_id_name_map = + G_RES_MGR.get_mapping_rule_mgr().get_group_id_name_map(); + ObResourceMappingRuleManager::GetTenantGroupIdNameFunctor functor(tenant_id, group_id_keys, group_names); + if (OB_FAIL(group_id_name_map.foreach_refactored(functor))) { + LOG_WARN("failed to do foreach", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < group_id_keys.count(); ++i) { + bool is_group_id_found = false; + for (int64_t j = 0; !is_group_id_found && j < directives.count(); ++j) { + const share::ObPlanDirective &cur_directive = directives.at(j); + if (cur_directive.group_id_ == group_id_keys.at(i).group_id_) { + is_group_id_found = true; + } + } + if (!is_group_id_found) { + const uint64_t deleted_group_id = group_id_keys.at(i).group_id_; + LOG_INFO("directive need to be cleared", K(tenant_id), K(deleted_group_id)); + if (OB_FAIL(tenant_holder.get_ptr()->reset_consumer_group_config(deleted_group_id))) { + LOG_WARN("reset consumer group config failed", K(ret), K(deleted_group_id)); + } else if (OB_FAIL(GCTX.cgroup_ctrl_->set_both_cpu_shares(tenant_id, + 1, + deleted_group_id, + GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) { + LOG_WARN("fail to set cpu share", K(ret), K(tenant_id), K(deleted_group_id)); + } else if (OB_FAIL(GCTX.cgroup_ctrl_->set_both_cpu_cfs_quota(tenant_id, + -1, + deleted_group_id, + GCONF.enable_global_background_resource_isolation ? BACKGROUND_CGROUP : ""))) { + LOG_WARN("fail to set cpu quota", K(ret), K(tenant_id), K(deleted_group_id)); + } + } + } + } } return ret; } diff --git a/src/share/resource_manager/ob_resource_plan_manager.h b/src/share/resource_manager/ob_resource_plan_manager.h index 51e83a847..417922da6 100644 --- a/src/share/resource_manager/ob_resource_plan_manager.h +++ b/src/share/resource_manager/ob_resource_plan_manager.h @@ -31,8 +31,6 @@ static constexpr int64_t OTHER_GROUPS_IOPS_WEIGHT = 100L; static constexpr int64_t OTHER_GROUPS_NET_BANDWIDTH_WEIGHT = 100L; class ObResourcePlanManager { -public: - typedef common::ObSEArray ObPlanDirectiveSet; public: ObResourcePlanManager() : tenant_plan_map_(), background_quota_(INT32_MAX) {} @@ -48,13 +46,13 @@ private: int flush_directive_to_iops_control(const uint64_t tenant_id, ObPlanDirectiveSet &directives, ObPlanDirective &other_group_directive); + int clear_deleted_directives(const uint64_t tenant_id, ObPlanDirectiveSet &directives); int normalize_iops_directives(const uint64_t tenant_id, ObPlanDirectiveSet &directives, ObPlanDirective &other_group_directive); int normalize_net_bandwidth_directives(const uint64_t tenant_id, ObPlanDirectiveSet &directives, ObPlanDirective &other_group_directive); - int refresh_tenant_group_io_config(const uint64_t tenant_id); common::hash::ObHashMap tenant_plan_map_; int32_t background_quota_; /* variables */