hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
src/sql/engine/ob_sql_mem_mgr_processor.cpp
View File

@ -21,18 +21,28 @@ using namespace omt;
namespace sql {
int ObSqlMemMgrProcessor::init(ObIAllocator* allocator, uint64_t tenant_id, int64_t cache_size,
const ObPhyOperatorType op_type, const uint64_t op_id, ObExecContext* exec_ctx)
int ObSqlMemMgrProcessor::init(
ObIAllocator *allocator,
uint64_t tenant_id,
int64_t cache_size,
const ObPhyOperatorType op_type,
const uint64_t op_id,
ObExecContext *exec_ctx)
{
int ret = OB_SUCCESS;
bool tmp_enable_auto_mem_mgr = false;
ObTenantSqlMemoryManager* sql_mem_mgr = get_sql_mem_mgr();
ObTenantSqlMemoryManager *sql_mem_mgr = get_sql_mem_mgr();
is_auto_mgr_ = false;
reset();
tenant_id_ = tenant_id;
profile_.set_operator_type(op_type);
profile_.set_operator_id(op_id);
profile_.set_exec_ctx(exec_ctx);
const int64_t DEFAULT_CACHE_SIZE = 2 * 1024 * 1024;
if (cache_size < 0) {
LOG_WARN("unexpected cache size got", K(lbt()), K(cache_size), K(op_id), K(op_type));
cache_size = DEFAULT_CACHE_SIZE;
}
if (OB_FAIL(alloc_dir_id(dir_id_))) {
} else if (OB_NOT_NULL(sql_mem_mgr)) {
if (sql_mem_mgr->enable_auto_memory_mgr()) {
@ -40,19 +50,21 @@ int ObSqlMemMgrProcessor::init(ObIAllocator* allocator, uint64_t tenant_id, int6
if (profile_.get_auto_policy()) {
// update
int64_t pre_size = profile_.get_cache_size();
if (cache_size != pre_size &&
OB_FAIL(sql_mem_mgr_->update_work_area_profile(allocator, profile_, cache_size - pre_size))) {
if (cache_size != pre_size
&& OB_FAIL(sql_mem_mgr_->update_work_area_profile(
allocator, profile_, cache_size - pre_size))) {
LOG_WARN("failed update work area profile", K(ret), K(cache_size));
} else {
profile_.init(cache_size, OB_MALLOC_MIDDLE_BLOCK_SIZE);
LOG_TRACE(
"trace update cache size", K(profile_.get_cache_size()), K(profile_.get_expect_size()), K(cache_size));
LOG_TRACE("trace update cache size", K(profile_.get_cache_size()),
K(profile_.get_expect_size()), K(cache_size));
}
} else {
// first time to register
profile_.init(cache_size, OB_MALLOC_MIDDLE_BLOCK_SIZE);
LOG_TRACE("trace register work area profile", K(profile_.get_cache_size()));
LOG_DEBUG("trace register work area profile", K(profile_.get_cache_size()));
}
// 每次初始化都认为是未注册
profile_.set_expect_size(OB_INVALID_ID);
if (OB_FAIL(ret)) {
} else if (OB_FAIL(sql_mem_mgr_->register_work_area_profile(profile_))) {
@ -61,12 +73,8 @@ int ObSqlMemMgrProcessor::init(ObIAllocator* allocator, uint64_t tenant_id, int6
LOG_WARN("failed to get available mem size", K(ret));
} else {
is_auto_mgr_ = profile_.get_auto_policy();
LOG_TRACE("trace enable sql memory manager",
K(ret),
K(profile_.get_cache_size()),
K(profile_.get_expect_size()),
K(is_auto_mgr_),
K(profile_.is_registered()));
LOG_DEBUG("trace enable sql memory manager", K(ret), K(profile_.get_cache_size()),
K(profile_.get_expect_size()), K(is_auto_mgr_), K(profile_.is_registered()));
}
} else {
profile_.init(cache_size, OB_MALLOC_MIDDLE_BLOCK_SIZE);
@ -74,30 +82,29 @@ int ObSqlMemMgrProcessor::init(ObIAllocator* allocator, uint64_t tenant_id, int6
if (OB_FAIL(sql_mem_mgr_->register_work_area_profile(profile_))) {
LOG_WARN("failed to register workarea profile", K(ret));
}
LOG_TRACE("trace only register sql memory manager",
K(ret),
K(profile_.get_cache_size()),
K(profile_.get_expect_size()),
K(is_auto_mgr_));
// 如果从AUTO->MANUAL，是否需要清理一些数据
LOG_TRACE("trace only register sql memory manager", K(ret), K(profile_.get_cache_size()),
K(profile_.get_expect_size()), K(is_auto_mgr_));
}
} else {
profile_.set_expect_size(OB_INVALID_ID);
}
int64_t max_mem_size = MAX_SQL_MEM_SIZE;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size(profile_.get_work_area_type(), tenant_id_, max_mem_size))) {
} else if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size(
profile_.get_work_area_type(), tenant_id_, max_mem_size))) {
LOG_WARN("failed to get workarea size", K(ret), K(tenant_id_), K(max_mem_size));
}
if (!profile_.get_auto_policy()) {
profile_.set_max_bound(max_mem_size);
}
// don't register small runners
// 如果开启了sql memory manager，但由于预估数据量比较少，不需要注册到manager里，这里限制为MAX_SQL_MEM_SIZE
origin_max_mem_size_ = max_mem_size;
default_available_mem_size_ = tmp_enable_auto_mem_mgr ? MAX_SQL_MEM_SIZE : max_mem_size;
return ret;
}
int ObSqlMemMgrProcessor::get_max_available_mem_size(ObIAllocator* allocator)
int ObSqlMemMgrProcessor::get_max_available_mem_size(ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_auto_mgr())) {
@ -105,14 +112,17 @@ int ObSqlMemMgrProcessor::get_max_available_mem_size(ObIAllocator* allocator)
if (OB_FAIL(sql_mem_mgr_->get_work_area_size(allocator, profile_))) {
LOG_WARN("failed to get work area size", K(ret));
} else {
LOG_TRACE("trace get max available mem size", K(profile_.get_cache_size()), K(profile_.get_expect_size()));
LOG_TRACE("trace get max available mem size",
K(profile_.get_cache_size()), K(profile_.get_expect_size()));
}
}
return ret;
}
int ObSqlMemMgrProcessor::update_max_available_mem_size_periodically(
ObIAllocator* allocator, PredFunc predicate, bool& updated)
ObIAllocator *allocator,
PredFunc predicate,
bool &updated)
{
int ret = OB_SUCCESS;
updated = false;
@ -122,16 +132,19 @@ int ObSqlMemMgrProcessor::update_max_available_mem_size_periodically(
} else {
updated = true;
periodic_cnt_ <<= 1;
LOG_TRACE("trace get max available mem size periodically",
K(periodic_cnt_),
K(profile_.get_cache_size()),
K(profile_.get_expect_size()));
if (0 >= periodic_cnt_) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("unexpected status: periodic_cnt_", K(periodic_cnt_));
}
LOG_TRACE("trace get max available mem size periodically", K(periodic_cnt_),
K(profile_.get_cache_size()), K(profile_.get_expect_size()));
}
}
return ret;
}
int ObSqlMemMgrProcessor::update_cache_size(ObIAllocator* allocator, int64_t cache_size)
// 调整cache size
int ObSqlMemMgrProcessor::update_cache_size(ObIAllocator *allocator, int64_t cache_size)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_auto_mgr())) {
@ -141,17 +154,20 @@ int ObSqlMemMgrProcessor::update_cache_size(ObIAllocator* allocator, int64_t cac
LOG_TRACE("unexpected status: profile need register", K(ret));
} else if (cache_size != pre_size) {
profile_.init(cache_size, OB_MALLOC_MIDDLE_BLOCK_SIZE);
if (OB_FAIL(sql_mem_mgr_->update_work_area_profile(allocator, profile_, cache_size - pre_size))) {
if (OB_FAIL(sql_mem_mgr_->update_work_area_profile(
allocator, profile_, cache_size - pre_size))) {
LOG_WARN("failed update work area profile", K(ret), K(cache_size));
} else {
LOG_TRACE(
"trace update cache size", K(profile_.get_cache_size()), K(profile_.get_expect_size()), K(cache_size));
LOG_TRACE("trace update cache size", K(profile_.get_cache_size()),
K(profile_.get_expect_size()), K(cache_size), K(pre_size));
}
}
}
return ret;
}
// 更新operator使用的memory size
// 这里只关注终态，即申请了多少，后续每次调用，则更新，最后一次需要减为0
int ObSqlMemMgrProcessor::update_used_mem_size(int64_t used_size)
{
int ret = OB_SUCCESS;
@ -175,22 +191,24 @@ int ObSqlMemMgrProcessor::update_used_mem_size(int64_t used_size)
return ret;
}
int ObSqlMemMgrProcessor::try_upgrade_auto_mgr(ObIAllocator* allocator, int64_t mem_used)
// 当auto模式打开，但由于"under-estimate"导致没有采用auto管理，内存设置了一个固定大小
// 如果超估了该值，会dump，这个时候采用"升级"模式，启动auto进行管理
int ObSqlMemMgrProcessor::try_upgrade_auto_mgr(ObIAllocator *allocator, int64_t mem_used)
{
int ret = OB_SUCCESS;
if (!is_auto_mgr() && OB_NOT_NULL(sql_mem_mgr_)) {
int64_t max_area_size = max(default_available_mem_size_, MAX_SQL_MEM_SIZE);
if (sql_mem_mgr_->enable_auto_memory_mgr() && sql_mem_mgr_->get_global_bound_size() < max_area_size) {
if (sql_mem_mgr_->enable_auto_memory_mgr()
&& sql_mem_mgr_->get_global_bound_size() < max_area_size) {
} else if (OB_FAIL(update_used_mem_size(0))) {
LOG_WARN("failed to update used mem size", K(ret));
} else if (OB_FAIL(init(allocator,
tenant_id_,
max_area_size * (EXTEND_RATIO + 100) / 100,
profile_.get_operator_type(),
profile_.get_operator_id(),
profile_.get_exec_ctx()))) {
} else if (OB_FAIL(init(allocator, tenant_id_,
max_area_size * (EXTEND_RATIO + 100) /100,
profile_.get_operator_type(),
profile_.get_operator_id(), profile_.get_exec_ctx()))) {
LOG_WARN("failed to upgrade sql memory manager", K(ret));
} else if (is_auto_mgr()) {
// 由于之前未注册，所以对内存统计不会反应到sql mem manager中，但现在注册了，则需要重新更新下内存值
if (OB_FAIL(update_used_mem_size(mem_used))) {
LOG_WARN("failed to update used mem_size", K(ret));
}
@ -204,7 +222,11 @@ int ObSqlMemMgrProcessor::try_upgrade_auto_mgr(ObIAllocator* allocator, int64_t
// try to extend max memory size to avoid dump or require more memory when need dump
int ObSqlMemMgrProcessor::extend_max_memory_size(
ObIAllocator* allocator, PredFunc dump_fun, bool& need_dump, int64_t mem_used, int64_t max_times)
ObIAllocator *allocator,
PredFunc dump_fun,
bool &need_dump,
int64_t mem_used,
int64_t max_times)
{
int ret = OB_SUCCESS;
need_dump = true;
@ -221,8 +243,12 @@ int ObSqlMemMgrProcessor::extend_max_memory_size(
int64_t pre_cache_size = profile_.get_cache_size();
int64_t pre_expect_size = profile_.get_expect_size();
if (pre_cache_size > pre_expect_size) {
// 内存管理给的的内存小于实际要求的
// 策略：获取当前最大可用内存，如果小于cache_size，则退出，说明内存管理模块只能给这么多
// 否则，继续判断是否dump，走真正扩展内存逻辑
if (OB_FAIL(get_max_available_mem_size(allocator))) {
LOG_WARN("failed to get max available memory size", K(pre_cache_size), K(pre_expect_size), K(ret));
LOG_WARN("failed to get max available memory size", K(pre_cache_size),
K(pre_expect_size), K(ret));
} else if (profile_.get_cache_size() > profile_.get_expect_size()) {
LOG_TRACE("trace extend max memory size", K(pre_cache_size), K(pre_expect_size));
break;
@ -232,27 +258,21 @@ int ObSqlMemMgrProcessor::extend_max_memory_size(
if (OB_FAIL(update_cache_size(allocator, new_cache_size))) {
LOG_WARN("failed to upadte cache size", K(ret), K(new_cache_size));
} else if (OB_FAIL(get_max_available_mem_size(allocator))) {
LOG_WARN("failed to get max available memory size",
K(new_cache_size),
K(pre_cache_size),
K(pre_expect_size),
K(ret));
LOG_WARN("failed to get max available memory size", K(new_cache_size),
K(pre_cache_size), K(pre_expect_size), K(ret));
} else if (profile_.get_cache_size() > profile_.get_expect_size()) {
if (OB_FAIL(update_cache_size(allocator, pre_cache_size))) {
LOG_WARN("failed to get max memory size", K(ret), K(pre_cache_size));
}
LOG_TRACE("trace extend max memory size",
K(pre_cache_size),
K(pre_expect_size),
K(profile_.get_cache_size()),
K(profile_.get_expect_size()));
LOG_TRACE("trace extend max memory size", K(pre_cache_size), K(pre_expect_size),
K(profile_.get_cache_size()), K(profile_.get_expect_size()));
break;
}
}
++times;
if (0 == times % 16) {
LOG_INFO(
"extend max memory size", K(ret), K(times), K(profile_.get_expect_size()), K(profile_.get_cache_size()));
LOG_INFO("extend max memory size", K(ret), K(times), K(profile_.get_expect_size()),
K(profile_.get_cache_size()));
}
if (max_times <= times) {
LOG_TRACE("extend memory size too more times", K(times));
@ -269,14 +289,13 @@ void ObSqlMemMgrProcessor::unregister_profile()
if (OB_NOT_NULL(sql_mem_mgr_)) {
sql_mem_mgr_->unregister_work_area_profile(profile_);
destroy();
LOG_TRACE("trace unregister work area profile", K(profile_));
LOG_DEBUG("trace unregister work area profile", K(profile_));
}
}
int ObSqlMemMgrProcessor::alloc_dir_id(int64_t& dir_id)
int ObSqlMemMgrProcessor::alloc_dir_id(int64_t &dir_id)
{
int ret = OB_SUCCESS;
dir_id = 0;
if (0 == dir_id_) {
if (OB_FAIL(ObChunkStoreUtil::alloc_dir_id(dir_id_))) {
LOG_WARN("failed to alloc dir id", K(ret));
@ -286,7 +305,7 @@ int ObSqlMemMgrProcessor::alloc_dir_id(int64_t& dir_id)
return ret;
}
int ObSqlWorkareaUtil::get_workarea_size(const ObSqlWorkAreaType wa_type, const int64_t tenant_id, int64_t& value)
int ObSqlWorkareaUtil::get_workarea_size(const ObSqlWorkAreaType wa_type, const int64_t tenant_id, int64_t &value)
{
int ret = OB_SUCCESS;
ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
@ -307,5 +326,13 @@ int ObSqlWorkareaUtil::get_workarea_size(const ObSqlWorkAreaType wa_type, const
return ret;
}
} // namespace sql
} // namespace oceanbase
void ObSqlMemMgrProcessor::unregister_profile_if_necessary()
{
if (!is_unregistered()) {
LOG_ERROR("profile is not actively unregistered", K(lbt()));
unregister_profile();
}
}
} // sql
} // oceanbase